Fixed endless processing

Ignore failing Ruling expansion
Pull request #165 : Fixed text parsing orientation problem
2021-06-02 13:47:48 +02:00 · 2021-06-02 12:53:30 +02:00 · 2021-06-02 10:58:00 +02:00 · 2021-06-02 10:47:40 +02:00 · 2021-06-01 13:47:59 +02:00 · 2021-06-01 13:41:22 +02:00
110 changed files with 3237 additions and 2170 deletions
@@ -5,7 +5,7 @@
  <parent>
    <groupId>com.atlassian.bamboo</groupId>
    <artifactId>bamboo-specs-parent</artifactId>
-    <version>7.1.2</version>
+    <version>7.2.2</version>
    <relativePath/>
  </parent>

@@ -1,7 +1,5 @@
 package buildjob;

-import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
-
 import com.atlassian.bamboo.specs.api.BambooSpec;
 import com.atlassian.bamboo.specs.api.builders.BambooKey;
 import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
@@ -23,6 +21,10 @@ import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
 import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
 import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
 import com.atlassian.bamboo.specs.util.BambooServer;
+import com.atlassian.bamboo.specs.builders.task.ScriptTask;
+import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
+
+import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;

 /**
 * Plan configuration for Bamboo.
@@ -33,6 +35,8 @@ public class PlanSpec {

    private static final String SERVICE_NAME = "redaction-service";

+    private static final String JVM_ARGS =" -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
+
    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");

    /**
@@ -82,20 +86,9 @@ public class PlanSpec {
                                                .checkoutItems(new CheckoutItem().defaultRepository()),
                                        new ScriptTask()
                                                .description("Build")
-                                                .inlineBody("#!/bin/bash\n" +
-                                                        "set -e\n" +
-
-                                                        "if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml versions:set  -DnewVersion=${bamboo.version_tag}; fi\n" +
-                                                        "if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml versions:set  -DnewVersion=${bamboo.version_tag}; fi\n" +
-
-                                                        "if [[ \"${bamboo.version_tag}\" = \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml --no-transfer-progress clean install -Djava.security.egd=file:/dev/./urandom; fi\n" +
-                                                        "if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml --no-transfer-progress clean deploy -e -DdeployAtEnd=true -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases; fi\n" +
-
-                                                        "${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml package\n" +
-                                                        "${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml docker:push\n" +
-
-                                                        "if [[ \"${bamboo.version_tag}\" = \"dev\" ]]; then echo \"gitTag=${bamboo.planRepository.1.branch}_${bamboo.buildNumber}\" > git.tag; fi\n" +
-                                                        "if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then echo \"gitTag=${bamboo.version_tag}\" > git.tag; fi\n"),
+                                                .location(Location.FILE)
+                                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                                .argument(SERVICE_NAME),
                                        createJUnitParserTask()
                                                .description("Resultparser")
                                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
@@ -0,0 +1,51 @@
+#!/bin/bash
+set -e
+
+SERVICE_NAME=$1
+
+if [[ "${bamboo_version_tag}" = "dev" ]]
+then 
+    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        --no-transfer-progress \
+        clean install \
+        -Djava.security.egd=file:/dev/./urandomelse
+else        
+    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+        --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        versions:set \
+        -DnewVersion=${bamboo_version_tag}
+    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+        --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+        versions:set  \
+        -DnewVersion=${bamboo_version_tag}
+    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        --no-transfer-progress \
+        clean deploy \
+        -e \
+        -DdeployAtEnd=true \
+        -Dmaven.wagon.http.ssl.insecure=true \
+        -Dmaven.wagon.http.ssl.allowall=true \
+        -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
+        -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
+fi
+
+${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+    --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    package
+
+${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+    --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    docker:push
+
+if [[ "${bamboo_version_tag}" = "dev" ]]
+then 
+    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
+else
+    echo "gitTag=${bamboo_version_tag}" > git.tag
+fi
@@ -5,7 +5,7 @@
    <parent>
        <artifactId>platform-dependency</artifactId>
        <groupId>com.iqser.red</groupId>
-        <version>1.0.8</version>
+        <version>1.1.2</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

@@ -32,7 +32,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.2.5</version>
+                <version>1.3.1</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -52,4 +52,4 @@

    </dependencyManagement>

-</project>
+</project>
@@ -17,5 +17,16 @@
            <artifactId>spring-web</artifactId>
            <optional>true</optional>
        </dependency>
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>configuration-service-api-v1</artifactId>
+            <version>2.11.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>file-management-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
    </dependencies>
 </project>
@@ -5,13 +5,20 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.time.OffsetDateTime;
+
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
 public class AnalyzeRequest {

-    private byte[] document;
-    private String ruleSetId;
+    private String dossierId;
+    private String fileId;
+    private String dossierTemplateId;
+    private boolean reanalyseOnlyIfPossible;
    private ManualRedactions manualRedactions;
+    private OffsetDateTime lastProcessed;
+
 }
+
@@ -11,9 +11,20 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
 public class AnalyzeResult {

+    private String dossierId;
+    private String fileId;
+    private long duration;
    private int numberOfPages;
-    private RedactionLog redactionLog;
-    private SectionGrid sectionGrid;
-    private Text text;
+    private boolean hasHints;
+    private boolean hasRequests;
+    private boolean hasRedactions;
+    private boolean hasImages;
+    private boolean hasUpdates;
+    private long dictionaryVersion;
+    private long dossierDictionaryVersion;
+    private long rulesVersion;
+    private long legalBasisVersion;
+

 }
+
@@ -11,7 +11,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
 public class AnnotateRequest {

-    private byte[] document;
-    private RedactionLog redactionLog;
-    private SectionGrid sectionGrid;
+    private String dossierId;
+    private String dossierTemplateId;
+    private String fileId;
 }
@@ -0,0 +1,5 @@
+package com.iqser.red.service.redaction.v1.model;
+
+public enum ChangeType {
+    ADDED, REMOVED
+}
@@ -1,12 +1,12 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.time.OffsetDateTime;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.time.OffsetDateTime;
+
@Data
@Builder
@AllArgsConstructor
@@ -1,13 +1,13 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.util.ArrayList;
-import java.util.List;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.util.ArrayList;
+import java.util.List;
+
@Data
@Builder
@AllArgsConstructor
@@ -27,4 +27,6 @@ public class ManualRedactionEntry {
    private String section;
    private int sectionNumber;

+    private boolean addToDossierDictionary;
+
 }
@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.model;

+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
@Data
@Builder
@AllArgsConstructor
@@ -0,0 +1,22 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class RedactionChangeLog {
+
+    private List<RedactionChangeLogEntry> redactionLogEntry = new ArrayList<>();
+
+    private long dictionaryVersion = -1;
+    private long dossierDictionaryVersion = -1;
+    private long rulesVersion = -1;
+    private long legalBasisVersion = -1;
+
+}
@@ -0,0 +1,47 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class RedactionChangeLogEntry {
+
+    private String id;
+    private String type;
+    private String value;
+    private String reason;
+    private int matchedRule;
+    private String legalBasis;
+    private boolean redacted;
+    private boolean isHint;
+    private boolean isRecommendation;
+    private String section;
+    private float[] color;
+
+    @Builder.Default
+    private List<Rectangle> positions = new ArrayList<>();
+    private int sectionNumber;
+    private boolean manual;
+    private Status status;
+    private ManualRedactionType manualRedactionType;
+    private boolean isDictionaryEntry;
+
+    private String textBefore;
+    private String textAfter;
+
+    @Builder.Default
+    private List<Comment> comments = new ArrayList<>();
+
+    private ChangeType changeType;
+
+    private boolean isDossierDictionaryEntry;
+
+}
@@ -1,32 +1,21 @@
 package com.iqser.red.service.redaction.v1.model;

+import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping;
 import lombok.AllArgsConstructor;
 import lombok.Data;
-import lombok.NoArgsConstructor;

 import java.util.List;

@Data
@AllArgsConstructor
-@NoArgsConstructor
 public class RedactionLog {

    private List<RedactionLogEntry> redactionLogEntry;
+    private List<LegalBasisMapping> legalBasis;

    private long dictionaryVersion = -1;
+    private long dossierDictionaryVersion = -1;
    private long rulesVersion = -1;
-
-    private String ruleSetId;
-    private String filename;
-
-
-    public RedactionLog(List<RedactionLogEntry> redactionLogEntry, long dictionaryVersion, long rulesVersion, String ruleSetId) {
-
-        this.redactionLogEntry = redactionLogEntry;
-        this.dictionaryVersion = dictionaryVersion;
-        this.rulesVersion = rulesVersion;
-        this.ruleSetId = ruleSetId;
-    }
-
+    private long legalBasisVersion = -1;

 }
@@ -1,17 +1,19 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.util.ArrayList;
-import java.util.List;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;

+import java.util.ArrayList;
+import java.util.List;
+
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
+@EqualsAndHashCode
 public class RedactionLogEntry {

    private String id;
@@ -45,4 +47,6 @@ public class RedactionLogEntry {

    private boolean isImage;

+    private boolean isDossierDictionaryEntry;
+
 }
@@ -11,7 +11,8 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
 public class RedactionRequest {

-    private byte[] document;
-    private String ruleSetId;
+    private String dossierId;
+    private String fileId;
+    private String dossierTemplateId;
    private ManualRedactions manualRedactions;
 }
@@ -13,7 +13,5 @@ public class RedactionResult {

    private byte[] document;
    private int numberOfPages;
-    private RedactionLog redactionLog;
-    private SectionGrid sectionGrid;

 }
@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.time.OffsetDateTime;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RenalyzeRequest {
-
-    private byte[] document;
-    private String ruleSetId;
-    private ManualRedactions manualRedactions;
-    private Text text;
-    private RedactionLog redactionLog;
-    private OffsetDateTime lastProcessed;
-}
@@ -27,7 +27,7 @@ public class SectionArea {
    private String header;

    public boolean contains(Rectangle other) {
-        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY()  && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight();
+        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight();
    }

 }
@@ -1,13 +1,13 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
@Data
@AllArgsConstructor
@NoArgsConstructor
@@ -1,13 +1,13 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.util.List;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;

+import java.util.List;
+
@Data
@AllArgsConstructor
@NoArgsConstructor
@@ -1,25 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class SectionText {
-
-    private int sectionNumber;
-    private String text;
-
-    private boolean isTable;
-    private String headline;
-
-    private List<SectionArea> sectionAreas = new ArrayList<>();
-
-}
@@ -1,14 +1,6 @@
 package com.iqser.red.service.redaction.v1.resources;

-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
-import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.model.ReanalyzeResult;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-import com.iqser.red.service.redaction.v1.model.RenalyzeRequest;
-
+import com.iqser.red.service.redaction.v1.model.*;
 import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -18,14 +10,9 @@ public interface RedactionResource {

    String SERVICE_NAME = "redaction-service-v1";

-    String RULE_SET_PARAMETER_NAME = "ruleSetId";
+    String RULE_SET_PARAMETER_NAME = "dossierTemplateId";
    String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}";

-    @PostMapping(value = "/analyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest);
-
-    @PostMapping(value = "/reanalyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    ReanalyzeResult reanalyze(@RequestBody RenalyzeRequest renalyzeRequest);

    @PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
    AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
@@ -39,10 +26,10 @@ public interface RedactionResource {
    @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
    RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);

-    @PostMapping(value = "/rules/update"+RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId);
+    @PostMapping(value = "/rules/update" + RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
+    void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId);

    @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
    void testRules(@RequestBody String rules);

-}
+}
@@ -12,6 +12,10 @@
    <artifactId>redaction-service-server-v1</artifactId>

    <dependencies>
+        <dependency>
+            <groupId>com.iqser.red.commons</groupId>
+            <artifactId>storage-commons</artifactId>
+        </dependency>
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>redaction-service-api-v1</artifactId>
@@ -19,8 +23,18 @@
        </dependency>
        <dependency>
            <groupId>com.iqser.red.service</groupId>
-            <artifactId>configuration-service-api-v1</artifactId>
-            <version>2.2.9</version>
+            <artifactId>file-management-service-api-v1</artifactId>
+            <version>2.25.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>redaction-service-api-v1</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>configuration-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.drools</groupId>
@@ -42,6 +56,23 @@
            <artifactId>guava</artifactId>
            <version>29.0-jre</version>
        </dependency>
+
+        <dependency>
+            <groupId>org.apache.pdfbox</groupId>
+            <artifactId>jbig2-imageio</artifactId>
+            <version>3.0.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.jai-imageio</groupId>
+            <artifactId>jai-imageio-core</artifactId>
+            <version>1.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.jai-imageio</groupId>
+            <artifactId>jai-imageio-jpeg2000</artifactId>
+            <version>1.4.0</version>
+        </dependency>
+
        <!-- commons -->
        <dependency>
            <groupId>com.iqser.red.commons</groupId>
@@ -74,6 +105,12 @@
            <artifactId>spring-cloud-starter-openfeign</artifactId>
        </dependency>

+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-amqp</artifactId>
+            <version>2.3.1.RELEASE</version>
+        </dependency>
+
        <!-- test dependencies -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
@@ -85,6 +122,12 @@
            <artifactId>test-commons</artifactId>
            <scope>test</scope>
        </dependency>
+        <dependency>
+            <groupId>org.springframework.amqp</groupId>
+            <artifactId>spring-rabbit-test</artifactId>
+            <version>2.3.1</version>
+            <scope>test</scope>
+        </dependency>
    </dependencies>

    <build>
@@ -1,5 +1,8 @@
 package com.iqser.red.service.redaction.v1.server;

+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -8,10 +11,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Import;

-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
@Import({DefaultWebMvcConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@@ -19,8 +18,9 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
 public class Application {

    public static void main(String[] args) {
+        System.setProperty("org.apache.pdfbox.rendering.UsePureJavaCMYKConversion", "true");
        SpringApplication.run(Application.class, args);
    }


-}
+}
@@ -1,20 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
@Data
@NoArgsConstructor
 public class Document {
@@ -33,7 +32,7 @@ public class Document {

    private List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
    private SectionGrid sectionGrid = new SectionGrid();
-    private long dictionaryVersion;
+    private DictionaryVersion dictionaryVersion;
    private long rulesVersion;

    private List<SectionText> sectionText = new ArrayList<>();
@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import lombok.Getter;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -7,38 +9,35 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;

-import lombok.Getter;
-
-public class FloatFrequencyCounter
-{
+public class FloatFrequencyCounter {

    @Getter
    Map<Float, Integer> countPerValue = new HashMap<>();

-    public void add(float value){
-        if(!countPerValue.containsKey(value)){
+    public void add(float value) {
+        if (!countPerValue.containsKey(value)) {
            countPerValue.put(value, 1);
        } else {
            countPerValue.put(value, countPerValue.get(value) + 1);
        }
    }

-    public void addAll(Map<Float, Integer> otherCounter){
-        for(Map.Entry<Float, Integer> entry: otherCounter.entrySet()){
-            if(countPerValue.containsKey(entry.getKey())){
-                countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey())+ entry.getValue());
+    public void addAll(Map<Float, Integer> otherCounter) {
+        for (Map.Entry<Float, Integer> entry : otherCounter.entrySet()) {
+            if (countPerValue.containsKey(entry.getKey())) {
+                countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
            } else {
                countPerValue.put(entry.getKey(), entry.getValue());
            }
        }
    }

-    public Float getMostPopular(){
+    public Float getMostPopular() {
        Map.Entry<Float, Integer> mostPopular = null;
-        for(Map.Entry<Float, Integer> entry: countPerValue.entrySet()){
-            if(mostPopular == null){
+        for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
+            if (mostPopular == null) {
                mostPopular = entry;
-            } else if(entry.getValue() >= mostPopular.getValue()){
+            } else if (entry.getValue() >= mostPopular.getValue()) {
                mostPopular = entry;
            }
        }
@@ -46,12 +45,11 @@ public class FloatFrequencyCounter
    }


-
-    public List<Float> getHighterThanMostPopular(){
+    public List<Float> getHighterThanMostPopular() {
        Float mostPopular = getMostPopular();
        List<Float> higher = new ArrayList<>();
-        for(Float value: countPerValue.keySet()){
-            if(value > mostPopular){
+        for (Float value : countPerValue.keySet()) {
+            if (value > mostPopular) {
                higher.add(value);
            }
        }
@@ -60,12 +58,12 @@ public class FloatFrequencyCounter
    }


-    public Float getHighest(){
+    public Float getHighest() {
        Float highest = null;
-        for(Float value: countPerValue.keySet()){
-            if (highest == null){
+        for (Float value : countPerValue.keySet()) {
+            if (highest == null) {
                highest = value;
-            } else if(value > highest){
+            } else if (value > highest) {
                highest = value;
            }
        }
@@ -1,19 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

-import java.util.List;
-
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;

+import java.util.List;
+
@Data
@AllArgsConstructor
 public class Footer {

    private List<TextBlock> textBlocks;

-
+    @JsonIgnore
    public SearchableText getSearchableText() {

        SearchableText searchableText = new SearchableText();
@@ -21,4 +21,4 @@ public class Footer {
        return searchableText;
    }

-}
+}
@@ -1,19 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

-import java.util.List;
-
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;

+import java.util.List;
+
@Data
@AllArgsConstructor
 public class Header {

    private List<TextBlock> textBlocks;

-
+    @JsonIgnore
    public SearchableText getSearchableText() {

        SearchableText searchableText = new SearchableText();
@@ -21,4 +21,4 @@ public class Header {
        return searchableText;
    }

-}
+}
@@ -0,0 +1,6 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+public enum Orientation {
+
+    NONE, LEFT, RIGHT
+}
@@ -1,15 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

-import java.util.List;
-
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
-
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;

+import java.util.List;
+
@Data
@RequiredArgsConstructor
 public class Page {
@@ -31,10 +30,12 @@ public class Page {
    private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
    private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();

+    private double cropBoxArea;
+

    public boolean isRotated() {

        return rotation != 0;
    }

-}
+}
@@ -1,19 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

-import java.util.ArrayList;
-import java.util.List;
-
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.util.ArrayList;
+import java.util.List;
+
@Data
@NoArgsConstructor
-public class Paragraph implements Comparable{
+public class Paragraph implements Comparable {

    private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
    private List<PdfImage> images = new ArrayList<>();
@@ -62,4 +61,4 @@ public class Paragraph implements Comparable{
        return 0;
    }

-}
+}
@@ -0,0 +1,52 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.*;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SectionText {
+
+    private int sectionNumber;
+    private String text;
+
+    private boolean isTable;
+    private String headline;
+
+    private List<SectionArea> sectionAreas = new ArrayList<>();
+    private Set<Image> images = new HashSet<>();
+
+    private List<TextBlock> textBlocks = new ArrayList<>();
+    private Map<String, CellValue> tabularData = new HashMap<>();
+    private List<Integer> cellStarts = new ArrayList<>();
+
+
+    public void setTabularData(Map<String, CellValue> tabularData) {
+        tabularData.remove(null);
+        this.tabularData = tabularData;
+    }
+
+    @JsonIgnore
+    public SearchableText getSearchableText() {
+
+        SearchableText searchableText = new SearchableText();
+        textBlocks.forEach(block -> {
+            if (block != null) {
+                searchableText.addAll(block.getSequences());
+            }
+        });
+        return searchableText;
+    }
+
+}
@@ -1,10 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import lombok.Getter;
+
 import java.util.HashMap;
 import java.util.Map;

-import lombok.Getter;
-
 public class StringFrequencyCounter {

    @Getter
@@ -46,4 +46,4 @@ public class StringFrequencyCounter {
        return mostPopular != null ? mostPopular.getKey() : null;
    }

-}
+}
@@ -1,17 +1,18 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.ArrayList;
-import java.util.List;
+package com.iqser.red.service.redaction.v1.server.classification.model;

 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.util.ArrayList;
+import java.util.List;
+
@Data
@NoArgsConstructor
@AllArgsConstructor
 public class Text {

+    private int numberOfPages;
    private List<SectionText> sectionTexts = new ArrayList<>();

 }
@@ -1,19 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
 import java.util.ArrayList;
 import java.util.List;

-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-
@AllArgsConstructor
@Builder
@Data
+@NoArgsConstructor
 public class TextBlock extends AbstractTextContainer {

    @Builder.Default
@@ -30,6 +32,7 @@ public class TextBlock extends AbstractTextContainer {

    private String classification;

+
    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
        this.minX = minX;
        this.maxX = maxX;
@@ -98,7 +101,6 @@ public class TextBlock extends AbstractTextContainer {
    }


-
    @Override
    public String toString() {

@@ -118,6 +120,7 @@ public class TextBlock extends AbstractTextContainer {
    }

    @Override
+    @JsonIgnore
    public String getText() {

        StringBuilder sb = new StringBuilder();
@@ -139,4 +142,4 @@ public class TextBlock extends AbstractTextContainer {

    }

-}
+}
@@ -1,19 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

-import java.util.List;
-
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;

+import java.util.List;
+
@Data
@AllArgsConstructor
 public class UnclassifiedText {

    private List<TextBlock> textBlocks;

-
+    @JsonIgnore
    public SearchableText getSearchableText() {

        SearchableText searchableText = new SearchableText();
@@ -21,4 +21,4 @@ public class UnclassifiedText {
        return searchableText;
    }

-}
+}
@@ -1,15 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;

-import java.util.ArrayList;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -17,11 +13,20 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;

+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
@Service
@SuppressWarnings("all")
 public class BlockificationService {

-    public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    static final float THRESHOLD = 1f;
+
+    public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines,
+                         List<Ruling> verticalRulingLines) {

        List<TextPositionSequence> chunkWords = new ArrayList<>();
        List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
@@ -29,21 +34,46 @@ public class BlockificationService {
        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
        TextPositionSequence prev = null;

+        boolean wasSplitted = false;
+        Float splitX1 = null;
        for (TextPositionSequence word : textPositions) {

            boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
            boolean startFromTop = word.getY1() > maxY + word.getHeight();
+            boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
+            boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
+            boolean splittedByRuling = word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines) || word
+                    .getRotation() == 0 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines) || word
+                    .getRotation() == 90 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines) || word
+                    .getRotation() == 90 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines);

-            if (prev != null && (lineSeparation || startFromTop || word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word
-                    .getX1(), word.getY1(), verticalRulingLines) || word.getRotation() == 0 && isSplittedByRuling(minX, minY, word
-                    .getX1(), word.getY2(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(maxX, minY, word
-                    .getX1(), word.getY1(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(minX, minY, word
-                    .getX1(), word.getY2(), verticalRulingLines))) {
+            if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling)) {
+
+                Orientation prevOrientation = null;
+                if(!chunkBlockList1.isEmpty()) {
+                    prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
+                }

                TextBlock cb1 = buildTextBlock(chunkWords);
                chunkBlockList1.add(cb1);
                chunkWords = new ArrayList<>();

+                if (splitByX && !splittedByRuling) {
+                    wasSplitted = true;
+                    cb1.setOrientation(Orientation.LEFT);
+                    splitX1 = word.getX1();
+                } else
+
+                if (newLineAfterSplit && !splittedByRuling) {
+                    wasSplitted = false;
+                    cb1.setOrientation(Orientation.RIGHT);
+                    splitX1 = null;
+                } else
+
+                if(prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (lineSeparation || !startFromTop || !splitByX || !newLineAfterSplit || !splittedByRuling)){
+                    cb1.setOrientation(Orientation.LEFT);
+                }
+
                minX = 1000;
                maxX = 0;
                minY = 1000;
@@ -73,9 +103,62 @@ public class BlockificationService {
            chunkBlockList1.add(cb1);
        }

+        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
+
+        TextBlock previousLeft = null;
+        TextBlock previousRight = null;
+        while (itty.hasNext()) {
+            TextBlock block = (TextBlock) itty.next();
+
+            if(previousLeft != null && block.getOrientation().equals(Orientation.LEFT)){
+                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()){
+                    previousLeft.add(block);
+                    itty.remove();
+                    continue;
+                }
+            }
+
+            if(previousRight != null && block.getOrientation().equals(Orientation.RIGHT)){
+                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()){
+                    previousRight.add(block);
+                    itty.remove();
+                    continue;
+                }
+            }
+
+            if (block.getOrientation().equals(Orientation.LEFT)) {
+                previousLeft = block;
+            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
+                previousRight = block;
+            }
+        }
+
+
+        itty = chunkBlockList1.iterator();
+        TextBlock previous = null;
+        while (itty.hasNext()) {
+            TextBlock block = (TextBlock) itty.next();
+
+            if(previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous
+                    .getMaxY())||
+                    previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous
+                    .getMaxY())){
+                previous.add(block);
+                itty.remove();
+                continue;
+            }
+
+            previous = block;
+        }
+
+
        return new Page(chunkBlockList1);
    }

+    private boolean equalsWithThreshold(float f1, float f2){
+        return Math.abs(f1 - f2) < THRESHOLD;
+    }
+

    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList) {

@@ -118,7 +201,8 @@ public class BlockificationService {
    }


-    private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines) {
+    private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1,
+                                       List<Ruling> rulingLines) {

        for (Ruling ruling : rulingLines) {
            if (ruling.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
@@ -129,7 +213,8 @@ public class BlockificationService {
    }


-    public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
+    public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter,
+                                            boolean landscape) {

        float minX = 10000;
        float maxX = -100;
@@ -1,19 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;

-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.regex.Pattern;

@Slf4j
@Service
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.classification.utils;

 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
-
 import lombok.experimental.UtilityClass;

@UtilityClass
@@ -0,0 +1,9 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+
+import com.iqser.red.service.file.management.v1.api.resources.FileStatusProcessingUpdateResource;
+import org.springframework.cloud.openfeign.FeignClient;
+
+@FeignClient(name = "FileStatusProcessingUpdateResource", url = "${file-management-service.url}")
+public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {
+}
@@ -0,0 +1,8 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import com.iqser.red.service.configuration.v1.api.resource.LegalBasisMappingResource;
+import org.springframework.cloud.openfeign.FeignClient;
+
+@FeignClient(name = "LegalBasisMappingResource", url = "${configuration-service.url}")
+public interface LegalBasisClient extends LegalBasisMappingResource {
+}
@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.client;

-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-
 import org.springframework.lang.NonNull;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 import org.springframework.util.FileCopyUtils;
 import org.springframework.web.multipart.MultipartFile;

+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
 public class MockMultipartFile implements MultipartFile {

    private final String name;
@@ -22,13 +22,13 @@ public class MockMultipartFile implements MultipartFile {

    public MockMultipartFile(String name, @Nullable byte[] content) {

-        this(name, "", (String) null, (byte[]) content);
+        this(name, "", null, content);
    }


    public MockMultipartFile(String name, InputStream contentStream) throws IOException {

-        this(name, "", (String) null, (byte[]) FileCopyUtils.copyToByteArray(contentStream));
+        this(name, "", null, FileCopyUtils.copyToByteArray(contentStream));
    }


@@ -78,7 +78,7 @@ public class MockMultipartFile implements MultipartFile {

    public long getSize() {

-        return (long) this.content.length;
+        return this.content.length;
    }


@@ -3,6 +3,6 @@ package com.iqser.red.service.redaction.v1.server.client;
 import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
 import org.springframework.cloud.openfeign.FeignClient;

-@FeignClient(name = RulesResource.SERVICE_NAME, url = "${configuration-service.url}")
+@FeignClient(name = "RulesResource", url = "${configuration-service.url}")
 public interface RulesClient extends RulesResource {
-}
+}
@@ -1,17 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.controller;

-import java.time.OffsetDateTime;
-
 import com.iqser.red.commons.spring.ErrorMessage;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpStatus;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestControllerAdvice;

-import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
-
-import lombok.extern.slf4j.Slf4j;
+import java.time.OffsetDateTime;

@Slf4j
@RestControllerAdvice
@@ -38,4 +36,4 @@ public class ControllerAdvice {
        return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
    }

-}
+}
@@ -1,17 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.controller;

-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.file.management.v1.api.model.FileType;
 import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.model.ReanalyzeResult;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
-import com.iqser.red.service.redaction.v1.model.RenalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.model.Text;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
@@ -19,27 +12,21 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ImageClassificationService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-
+import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;

-import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.List;

@Slf4j
@RestController
@@ -47,61 +34,24 @@ import java.util.List;
 public class RedactionController implements RedactionResource {

    private final PdfVisualisationService pdfVisualisationService;
-    private final PdfSegmentationService pdfSegmentationService;
-    private final RedactionLogCreatorService redactionLogCreatorService;
-    private final EntityRedactionService entityRedactionService;
    private final DroolsExecutionService droolsExecutionService;
    private final DictionaryService dictionaryService;
    private final AnnotationService annotationService;
-    private final ReanalyzeService reanalyzeService;
-    private final ImageClassificationService imageClassificationService;
-
-
-    @Override
-    public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
-
-        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
-            pdDocument.setAllSecurityToBeRemoved(true);
-
-            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
-
-            log.info("Document structure analysis successful, starting redaction analysis...");
-
-            imageClassificationService.classifyImages(classifiedDoc);
-            entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getRuleSetId(), analyzeRequest.getManualRedactions());
-            redactionLogCreatorService.createRedactionLog(classifiedDoc, pdDocument.getNumberOfPages(), analyzeRequest.getManualRedactions(), analyzeRequest
-                    .getRuleSetId());
-
-            log.info("Redaction analysis successful...");
-
-            return AnalyzeResult.builder()
-                    .sectionGrid(classifiedDoc.getSectionGrid())
-                    .redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
-                            .getRulesVersion(), analyzeRequest.getRuleSetId()))
-                    .numberOfPages(classifiedDoc.getPages().size())
-                    .text(new Text(classifiedDoc.getSectionText()))
-                    .build();
-
-        } catch (Exception e) {
-            throw new RedactionException(e);
-        }
-
-    }
-
-
-    public ReanalyzeResult reanalyze(@RequestBody RenalyzeRequest renalyzeRequest) {
-
-        return reanalyzeService.reanalyze(renalyzeRequest);
-    }
+    private final PdfSegmentationService pdfSegmentationService;
+    private final RedactionStorageService redactionStorageService;


    public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {

-        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(annotateRequest.getDocument()))) {
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
+        var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
+        var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());

+        try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
            pdDocument.setAllSecurityToBeRemoved(true);
-            dictionaryService.updateDictionary(annotateRequest.getRedactionLog().getRuleSetId());
-            annotationService.annotate(pdDocument, annotateRequest.getRedactionLog(), annotateRequest.getSectionGrid());
+
+            dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
+            annotationService.annotate(pdDocument, redactionLog, sectionsGrid);

            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
                pdDocument.save(byteArrayOutputStream);
@@ -115,72 +65,87 @@ public class RedactionController implements RedactionResource {


    @Override
-    public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {
+    public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+        try {
+            Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);

-        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
-            pdDocument.setAllSecurityToBeRemoved(true);
+            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
+                pdDocument.setAllSecurityToBeRemoved(true);

-            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
-            pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
+                pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);

-            return convert(pdDocument, classifiedDoc.getPages().size(), pdfSegmentationRequest.getRuleSetId());
+                return convert(pdDocument, classifiedDoc.getPages().size());
+
+            } catch (IOException e) {
+                throw new RedactionException(e);
+            }

        } catch (IOException e) {
            throw new RedactionException(e);
        }

+
    }

-
    @Override
    public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+        try {
+            Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);

-        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
-            pdDocument.setAllSecurityToBeRemoved(true);
+            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
+                pdDocument.setAllSecurityToBeRemoved(true);

-            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
-            pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
+                pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
+                return convert(pdDocument, classifiedDoc.getPages().size());

-            return convert(pdDocument, classifiedDoc.getPages().size(), redactionRequest.getRuleSetId());
+            } catch (IOException e) {
+                throw new RedactionException(e);
+            }

        } catch (IOException e) {
            throw new RedactionException(e);
        }

+
    }


    @Override
    public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {

-        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
-            pdDocument.setAllSecurityToBeRemoved(true);
+        Document classifiedDoc;

-            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
-
-            StringBuilder sb = new StringBuilder();
-            for (Page page : classifiedDoc.getPages()) {
-                for (AbstractTextContainer textContainer : page.getTextBlocks()) {
-                    if (textContainer instanceof Table) {
-                        Table table = (Table) textContainer;
-                        sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
-                    }
-                }
-            }
-
-            return RedactionResult.builder().document(sb.toString().getBytes()).build();
-
-        } catch (IOException e) {
+        try {
+            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, true);
+        } catch (Exception e) {
            throw new RedactionException(e);
        }

+
+        StringBuilder sb = new StringBuilder();
+        for (Page page : classifiedDoc.getPages()) {
+            for (AbstractTextContainer textContainer : page.getTextBlocks()) {
+                if (textContainer instanceof Table) {
+                    Table table = (Table) textContainer;
+                    sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
+                }
+            }
+        }
+
+        return RedactionResult.builder().document(sb.toString().getBytes()).build();
+
    }


    @Override
-    public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId) {
+    public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId) {

-        droolsExecutionService.updateRules(ruleSetId);
+        droolsExecutionService.updateRules(dossierTemplateId);
    }


@@ -191,26 +156,17 @@ public class RedactionController implements RedactionResource {
    }


-    private RedactionResult convert(PDDocument document, int numberOfPages, String ruleSetId) throws IOException {
-
-        return convert(document, numberOfPages, null, null, 0, 0, ruleSetId);
-    }
-
-
-    private RedactionResult convert(PDDocument document, int numberOfPages,
-                                    List<RedactionLogEntry> redactionLogEntities, SectionGrid sectionGrid,
-                                    long dictionaryVersion, long rulesVersion, String ruleSetId) throws IOException {
+    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {

        try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
            document.save(byteArrayOutputStream);
            return RedactionResult.builder()
                    .document(byteArrayOutputStream.toByteArray())
                    .numberOfPages(numberOfPages)
-                    .redactionLog(new RedactionLog(redactionLogEntities, dictionaryVersion, rulesVersion, ruleSetId))
-                    .sectionGrid(sectionGrid)
                    .build();
        }

    }

+
 }
@@ -0,0 +1,52 @@
+package com.iqser.red.service.redaction.v1.server.memory;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.CharacterIterator;
+import java.text.StringCharacterIterator;
+
+@Slf4j
+public class MemoryStats {
+
+
+    public static void printMemoryStats() {
+        log.info("\n\n ------------------------------ \n" +
+                " Used Memory: " + humanReadableByteCountBin(getUsedMemory()) + "\n" +
+                " Free Memory: " + humanReadableByteCountBin(getFreeMemory()) + "\n" +
+                " Total Memory: " + humanReadableByteCountBin(getTotalMemory()) + "\n" +
+                " Max Memory: " + humanReadableByteCountBin(getMaxMemory()) + "\n" +
+                "\n ------------------------------ \n");
+    }
+
+
+    public static String humanReadableByteCountBin(long bytes) {
+        long absB = bytes == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(bytes);
+        if (absB < 1024) {
+            return bytes + " B";
+        }
+        long value = absB;
+        CharacterIterator ci = new StringCharacterIterator("KMGTPE");
+        for (int i = 40; i >= 0 && absB > 0xfffccccccccccccL >> i; i -= 10) {
+            value >>= 10;
+            ci.next();
+        }
+        value *= Long.signum(bytes);
+        return String.format("%.1f %ciB", value / 1024.0, ci.current());
+    }
+
+    private static long getMaxMemory() {
+        return Runtime.getRuntime().maxMemory();
+    }
+
+    private static long getUsedMemory() {
+        return getMaxMemory() - getFreeMemory();
+    }
+
+    private static long getTotalMemory() {
+        return Runtime.getRuntime().totalMemory();
+    }
+
+    private static long getFreeMemory() {
+        return Runtime.getRuntime().freeMemory();
+    }
+}
@@ -1,17 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.parsing;

+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.pdfbox.text.PDFTextStripperByArea;
+import org.apache.pdfbox.text.TextPosition;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;

-import org.apache.pdfbox.text.PDFTextStripperByArea;
-import org.apache.pdfbox.text.TextPosition;
-
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.Getter;
-import lombok.Setter;
-
 public class PDFAreaTextStripper extends PDFTextStripperByArea {

    @Getter
@@ -48,6 +46,17 @@ public class PDFAreaTextStripper extends PDFTextStripperByArea {
                startIndex = i;
            }

+
+            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                }
+                startIndex = i;
+            }
+
            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
                    .getUnicode()
                    .equals("\u00A0")) && i <= textPositions.size() - 2) {
@@ -76,7 +85,7 @@ public class PDFAreaTextStripper extends PDFTextStripperByArea {
    }


-    public void clearPositions(){
+    public void clearPositions() {
        textPositionSequences = new ArrayList<>();
    }

@@ -1,33 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.parsing;

-import java.awt.geom.AffineTransform;
-import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
-import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
-import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
-import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
-import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
+import org.apache.pdfbox.contentstream.operator.color.*;
+import org.apache.pdfbox.contentstream.operator.state.*;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSName;
@@ -40,40 +23,31 @@ import org.apache.pdfbox.text.PDFTextStripper;
 import org.apache.pdfbox.text.TextPosition;
 import org.apache.pdfbox.util.Matrix;

-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
-
-import lombok.Getter;
-import lombok.Setter;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;

@Slf4j
 public class PDFLinesTextStripper extends PDFTextStripper {

-    @Setter
-    protected PDPage pdpage;
-
-    @Getter
-    private int minCharWidth;
-
-    @Getter
-    private int maxCharWidth;
-
-    @Getter
-    private int minCharHeight;
-
-    @Getter
-    private int maxCharHeight;
-
    @Getter
    private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-
    @Getter
    private final List<Ruling> rulings = new ArrayList<>();
-
    private final List<Ruling> graphicsPath = new ArrayList<>();
-
+    @Setter
+    protected PDPage pdpage;
+    @Getter
+    private int minCharWidth;
+    @Getter
+    private int maxCharWidth;
+    @Getter
+    private int minCharHeight;
+    @Getter
+    private int maxCharHeight;
    @Getter
    private List<PdfImage> images = new ArrayList<>();

@@ -222,6 +196,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds
                        .getWidth(), (float) imageBounds.getHeight());

+                // Memory Hack - sofReference kills me
+                FieldUtils.writeField(pdfImage, "cachedImageSubsampling", -1, true);
+
                if (rect.getHeight() > 2 && rect.getWidth() > 2) {
                    this.images.add(new PdfImage(pdfImage.getImage(), rect, pageNumber));
                }
@@ -323,6 +300,18 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                startIndex = i;
            }

+
+            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                }
+                startIndex = i;
+            }
+
+
            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
                    .getUnicode()
                    .equals("\u00A0")) && i <= textPositions.size() - 2) {
@@ -369,4 +358,4 @@ public class PDFLinesTextStripper extends PDFTextStripper {

    }

-}
+}
@@ -0,0 +1,52 @@
+package com.iqser.red.service.redaction.v1.server.parsing.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.SneakyThrows;
+import org.apache.pdfbox.text.TextPosition;
+import org.springframework.beans.BeanUtils;
+
+@Data
+@NoArgsConstructor
+public class RedTextPosition {
+
+    private String textMatrix;
+    private int rotation;
+    private float y;
+    private float pageHeight;
+    private float pageWidth;
+    private String unicode;
+    private float XDirAdj;
+    private float YDirAdj;
+    private float width;
+    private float heightDir;
+
+    // not used in reanalysis
+    @JsonIgnore
+    private float widthOfSpace;
+
+    // not used in reanalysis
+    @JsonIgnore
+    private float fontSizeInPt;
+
+    // not used in reanalysis
+    @JsonIgnore
+    private String fontName;
+
+
+    @SneakyThrows
+    public static RedTextPosition fromTextPosition(TextPosition textPosition) {
+        var pos = new RedTextPosition();
+        BeanUtils.copyProperties(textPosition, pos);
+        pos.setFontName(textPosition.getFont().getName());
+
+        pos.setFontSizeInPt(textPosition.getFontSizeInPt());
+
+        pos.setTextMatrix(textPosition.getTextMatrix().toString());
+
+        return pos;
+    }
+
+
+}
@@ -1,32 +1,52 @@
 package com.iqser.red.service.redaction.v1.server.parsing.model;

+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.pdfbox.text.TextPosition;
+
 import java.util.ArrayList;
 import java.util.List;
-
-import org.apache.pdfbox.text.TextPosition;
-
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
+import java.util.stream.Collectors;

@Data
-@RequiredArgsConstructor
+@NoArgsConstructor
+@JsonIgnoreProperties({ "empty" })
 public class TextPositionSequence implements CharSequence {

-    private List<TextPosition> textPositions = new ArrayList<>();
+    private int page;
+    private List<RedTextPosition> textPositions = new ArrayList<>();

-    private final int page;
+    private float x1;
+    private float x2;
+
+    public TextPositionSequence(int page) {
+        this.page = page;
+    }
+
+
+    public static TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
+        var textPositionSequence = new TextPositionSequence();
+        textPositionSequence.textPositions = textPositions;
+        textPositionSequence.page = page;
+
+        return textPositionSequence;
+    }


    public TextPositionSequence(List<TextPosition> textPositions, int page) {

-        this.textPositions = textPositions;
+        this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
        this.page = page;
    }


+
+
+
    @Override
    public int length() {

@@ -37,7 +57,7 @@ public class TextPositionSequence implements CharSequence {
    @Override
    public char charAt(int index) {

-        TextPosition textPosition = textPositionAt(index);
+        RedTextPosition textPosition = textPositionAt(index);
        String text = textPosition.getUnicode();
        return text.charAt(0);
    }
@@ -45,7 +65,7 @@ public class TextPositionSequence implements CharSequence {

    public char charAt(int index, boolean caseInSensitive) {

-        TextPosition textPosition = textPositionAt(index);
+        RedTextPosition textPosition = textPositionAt(index);
        String text = textPosition.getUnicode();
        return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
    }
@@ -54,7 +74,7 @@ public class TextPositionSequence implements CharSequence {
    @Override
    public TextPositionSequence subSequence(int start, int end) {

-        return new TextPositionSequence(textPositions.subList(start, end), page);
+        return fromData(textPositions.subList(start, end), page);
    }


@@ -69,18 +89,25 @@ public class TextPositionSequence implements CharSequence {
    }


-    public TextPosition textPositionAt(int index) {
+    public RedTextPosition textPositionAt(int index) {

        return textPositions.get(index);
    }


-    public void add(TextPosition textPosition) {
+    public void add(RedTextPosition textPosition) {

        this.textPositions.add(textPosition);
    }


+    public void add(TextPosition textPosition) {
+
+        this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
+    }
+
+
+    @JsonIgnore
    public float getX1() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -91,6 +118,7 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
    public float getX2() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -101,13 +129,14 @@ public class TextPositionSequence implements CharSequence {
        }
    }

-
+    @JsonIgnore
    public float getRotationAdjustedY() {

        return textPositions.get(0).getY();
    }


+    @JsonIgnore
    public float getY1() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -118,6 +147,7 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
    public float getY2() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -128,38 +158,40 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + 2;
    }


+    @JsonIgnore
    public float getHeight() {

        return getY2() - getY1();
    }


+    @JsonIgnore
    public float getWidth() {

        return getX2() - getX1();
    }


+    @JsonIgnore
    public String getFont() {
-
-        return textPositions.get(0)
-                .getFont()
-                .toString()
+        return textPositions.get(0).getFontName()
                .toLowerCase()
                .replaceAll(",bold", "")
                .replaceAll(",italic", "");
    }


+    @JsonIgnore
    public String getFontStyle() {

-        String lowercaseFontName = textPositions.get(0).getFont().toString().toLowerCase();
+        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();

        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
            return "bold, italic";
@@ -173,25 +205,25 @@ public class TextPositionSequence implements CharSequence {

    }

-
+    @JsonIgnore
    public float getFontSize() {

        return textPositions.get(0).getFontSizeInPt();
    }

-
+    @JsonIgnore
    public float getSpaceWidth() {

        return textPositions.get(0).getWidthOfSpace();
    }

-
+    @JsonIgnore
    public int getRotation() {

        return textPositions.get(0).getRotation();
    }

-
+    @JsonIgnore
    public Rectangle getRectangle() {

        float height = getTextHeight();
@@ -223,4 +255,4 @@ public class TextPositionSequence implements CharSequence {
        return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
    }

-}
+}
@@ -0,0 +1,35 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@RequiredArgsConstructor
+public class MessagingConfiguration {
+
+
+    public static final String REDACTION_QUEUE = "redactionQueue";
+
+    public static final String REDACTION_DQL = "redactionDQL";
+
+
+    @Bean
+    public Queue redactionQueue() {
+
+        return QueueBuilder.durable(REDACTION_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
+                .maxPriority(2)
+                .build();
+    }
+
+
+    @Bean
+    public Queue redactionDeadLetterQueue() {
+
+        return QueueBuilder.durable(REDACTION_DQL).build();
+    }
+}
@@ -0,0 +1,54 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class RedactionMessageReceiver {
+
+    private final ObjectMapper objectMapper;
+    private final ReanalyzeService reanalyzeService;
+    private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
+
+    @RabbitHandler
+    @RabbitListener(queues = REDACTION_QUEUE)
+    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+
+        var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
+        log.info("Processing analyze request: {}", analyzeRequest);
+        AnalyzeResult result;
+        if (analyzeRequest.isReanalyseOnlyIfPossible()) {
+            result = reanalyzeService.reanalyze(analyzeRequest);
+        } else {
+            result = reanalyzeService.analyze(analyzeRequest);
+        }
+        log.info("Successfully analyzed {}", analyzeRequest);
+
+        fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
+    }
+
+    @RabbitHandler
+    @RabbitListener(queues = REDACTION_DQL)
+    public void receiveAnalyzeRequestDQL(String in) throws JsonProcessingException {
+
+        var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
+        log.info("Failed to process analyze request: {}", analyzeRequest);
+
+        fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+    }
+
+}
@@ -1,22 +1,25 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.util.Iterator;
-import java.util.List;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;

-import lombok.Value;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;

-@Value
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
 public class CellValue {

-    private List<TextBlock> textBlocks;
+    private List<TextBlock> textBlocks = new ArrayList<>();

    private int rowSpanStart;

-
    @Override
    public String toString() {

@@ -47,4 +50,4 @@ public class CellValue {
                .replaceAll(" {2}", " ");
    }

-}
+}
@@ -1,13 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import lombok.Data;
+import lombok.Getter;
+
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

-import lombok.Data;
-import lombok.Getter;
-
@Data
 public class Dictionary {

@@ -18,18 +18,18 @@ public class Dictionary {
    private Map<String, DictionaryModel> localAccessMap = new HashMap<>();

    @Getter
-    private long version;
+    private DictionaryVersion version;


-    public Dictionary(List<DictionaryModel> dictionaryModels, long dictionaryVersion){
+    public Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version) {
        this.dictionaryModels = dictionaryModels;
        this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
-        this.version = dictionaryVersion;
+        this.version = version;
    }


-    public int getDictionaryRank(String type){
-        if(!localAccessMap.containsKey(type)){
+    public int getDictionaryRank(String type) {
+        if (!localAccessMap.containsKey(type)) {
            return 0;
        }
        return localAccessMap.get(type).getRank();
@@ -60,7 +60,7 @@ public class Dictionary {

    public boolean containsValue(String type, String value) {

-        if (localAccessMap.containsKey(type) && localAccessMap.get(type)
+        return localAccessMap.containsKey(type) && localAccessMap.get(type)
                .getEntries()
                .contains(value) || localAccessMap.containsKey(type) && localAccessMap.get(type)
                .getLocalEntries()
@@ -68,10 +68,7 @@ public class Dictionary {
                .getEntries()
                .contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
                .getLocalEntries()
-                .contains(value)) {
-            return true;
-        }
-        return false;
+                .contains(value);
    }


@@ -1,15 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.util.Set;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;

+import java.util.Set;
+
@Data
@AllArgsConstructor
 public class DictionaryIncrement {

    private Set<DictionaryIncrementValue> values;
-    private long dictionaryVersion;
+    private DictionaryVersion dictionaryVersion;

 }
@@ -1,15 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;


+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
 import java.io.Serializable;
 import java.util.Set;
 import java.util.stream.Collectors;

-import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
@Data
@AllArgsConstructor
 public class DictionaryModel implements Serializable {
@@ -22,9 +21,10 @@ public class DictionaryModel implements Serializable {
    private boolean recommendation;
    private Set<DictionaryEntry> entries;
    private Set<String> localEntries;
+    private boolean isDossierDictionary;

-    public Set<String> getValues(boolean local){
-        return local ? localEntries : entries.stream().filter(e -> !e.isDeleted()).map(e-> e.getValue()).collect(Collectors
+    public Set<String> getValues(boolean local) {
+        return local ? localEntries : entries.stream().filter(e -> !e.isDeleted()).map(e -> e.getValue()).collect(Collectors
                .toSet());
    }

@@ -10,7 +10,7 @@ import java.util.Map;
@Data
 public class DictionaryRepresentation {

-    private String ruleSetId;
+    private String dossierTemplateId;
    private long dictionaryVersion = -1;
    private List<DictionaryModel> dictionary = new ArrayList<>();
    private float[] defaultColor;
@@ -20,5 +20,4 @@ public class DictionaryRepresentation {
    private Map<String, DictionaryModel> localAccessMap = new HashMap<>();


-
 }
@@ -0,0 +1,16 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DictionaryVersion {
+
+    long dossierTemplateVersion;
+    long dossierVersion;
+}
@@ -1,13 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
 import java.util.ArrayList;
 import java.util.List;

-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-
@Data
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity {
@@ -38,8 +37,10 @@ public class Entity {
    private String textBefore;
    private String textAfter;

+    private boolean isDossierDictionaryEntry;

-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end) {
+
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end, boolean isDossierDictionaryEntry) {

        this.word = word;
        this.type = type;
@@ -55,10 +56,11 @@ public class Entity {
        this.textAfter = textAfter;
        this.start = start;
        this.end = end;
+        this.isDossierDictionaryEntry = isDossierDictionaryEntry;
    }


-    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry) {
+    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionaryEntry) {

        this.word = word;
        this.type = type;
@@ -67,6 +69,7 @@ public class Entity {
        this.headline = headline;
        this.sectionNumber = sectionNumber;
        this.isDictionaryEntry = isDictionaryEntry;
+        this.isDossierDictionaryEntry = isDossierDictionaryEntry;
    }

 }
@@ -1,24 +1,23 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.util.ArrayList;
-import java.util.List;
-
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;

+import java.util.ArrayList;
+import java.util.List;
+
@Data
@RequiredArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode
 public class EntityPositionSequence {

+    private final String id;
    @EqualsAndHashCode.Exclude
    private List<TextPositionSequence> sequences = new ArrayList<>();
    private int pageNumber;
-    private final String id;

 }
@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.awt.geom.Rectangle2D;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -14,7 +12,7 @@ import lombok.NoArgsConstructor;
 public class Image {

    private String type;
-    private Rectangle2D position;
+    private RedRectangle2D position;
    private boolean redaction;
    private String redactionReason;
    private String legalBasis;
@@ -1,28 +1,31 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Data;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+
 import java.awt.geom.Rectangle2D;
 import java.awt.image.BufferedImage;

-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-
@Data
-@NoArgsConstructor
-@AllArgsConstructor
@RequiredArgsConstructor
 public class PdfImage {

-    @NonNull
+    @JsonIgnore
    private BufferedImage image;
    @NonNull
-    private Rectangle2D position;
+    private RedRectangle2D position;
    private ImageType imageType;
    private boolean isAppendedToParagraph;

    @NonNull
    private int page;

-}
+    public PdfImage(BufferedImage image, Rectangle2D position, int page) {
+        this.image = image;
+        this.position = new RedRectangle2D(position.getX(), position.getY(), position.getWidth(), position.getHeight());
+        this.page = page;
+    }
+
+}
@@ -1,37 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.model;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-public class ReanalysisSection {
-
-    private int sectionNumber;
-    private String headline;
-    private List<TextBlock> textBlocks;
-    private Map<String, CellValue> tabularData = new HashMap<>();
-    private List<Integer> cellStarts;
-    private Set<Image> images = new HashSet<>();
-
-
-    public SearchableText getSearchableText() {
-
-        SearchableText searchableText = new SearchableText();
-        textBlocks.forEach(block -> {
-            if (block instanceof TextBlock) {
-                searchableText.addAll(block.getSequences());
-            }
-        });
-        return searchableText;
-    }
-
-}
@@ -0,0 +1,42 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class RedRectangle2D {
+
+    public static final double THRESHOLD = 0.01;
+
+    private double x;
+    private double y;
+    private double width;
+    private double height;
+
+    @JsonIgnore
+    public boolean isEmpty() {
+        return width <= 0.0f || height <= 0.0f;
+    }
+
+    public boolean contains(double x, double y, double w, double h) {
+        if (isEmpty() || w <= 0 || h <= 0) {
+            return false;
+        }
+        double x0 = getX();
+        double y0 = getY();
+        return round(x) >= round(x0) &&
+                round(y) >= round(y0) &&
+                (x + w) - (x0 + getWidth()) <= THRESHOLD &&
+                (y + h) - (y0 + getHeight()) <= THRESHOLD;
+    }
+
+    private double round(double value) {
+        double d = Math.pow(10, 2);
+        return Math.round(value * d) / d;
+    }
+}
@@ -1,14 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.regex.Pattern;

-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-
 public class SearchableText {

    private final List<TextPositionSequence> sequences = new ArrayList<>();
@@ -232,4 +232,4 @@ public class SearchableText {
        return sb.append("\n").toString();
    }

-}
+}
@@ -1,6 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;

 import java.util.Collection;
 import java.util.HashMap;
@@ -11,15 +17,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;

-import org.apache.commons.lang3.StringUtils;
-
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
-
-import lombok.Builder;
-import lombok.Data;
-import lombok.extern.slf4j.Slf4j;
+import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;

@Data
@Slf4j
@@ -413,7 +411,7 @@ public class Section {
        String text = caseInsensitive ? searchText.toLowerCase() : searchText;
        String searchValue = caseInsensitive ? value.toLowerCase() : value;

-        Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, true);
+        Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, true, false);

        found.forEach(entity -> {
            if (redacted) {
@@ -439,7 +437,7 @@ public class Section {
        } else {
            String word = value.toString();

-            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false);
+            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false, false);
            entity.setRedaction(redact);
            entity.setMatchedRule(ruleNumber);
            entity.setRedactionReason(reason);
@@ -0,0 +1,49 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import org.springframework.stereotype.Service;
+
+@Service
+public class AnalyzeResponseService {
+
+    public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) {
+        boolean hasHints = redactionLog.getRedactionLogEntry().stream().anyMatch(RedactionLogEntry::isHint);
+
+        boolean hasRequests = redactionLog.getRedactionLogEntry()
+                .stream()
+                .anyMatch(entry -> entry.isManual() && entry.getStatus()
+                        .equals(com.iqser.red.service.redaction.v1.model.Status.REQUESTED));
+
+        boolean hasRedactions = redactionLog.getRedactionLogEntry()
+                .stream()
+                .anyMatch(entry -> entry.isRedacted() && !entry.isManual() || entry.isManual() && entry.getStatus()
+                        .equals(com.iqser.red.service.redaction.v1.model.Status.APPROVED));
+
+        boolean hasImages = redactionLog.getRedactionLogEntry()
+                .stream()
+                .anyMatch(entry -> entry.isHint() && entry.getType().equals("image"));
+
+        boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog
+                .getRedactionLogEntry()
+                .isEmpty() && redactionChangeLog.getRedactionLogEntry().stream().anyMatch(entry -> !entry.getType().equals("false_positive"));
+
+        return AnalyzeResult.builder()
+                .dossierId(dossierId)
+                .fileId(fileId)
+                .duration(duration)
+                .numberOfPages(pageCount)
+                .hasHints(hasHints)
+                .hasRedactions(hasRedactions)
+                .hasRequests(hasRequests)
+                .hasImages(hasImages)
+                .hasUpdates(hasUpdates)
+                .rulesVersion(redactionLog.getRulesVersion())
+                .dictionaryVersion(redactionLog.getDictionaryVersion())
+                .legalBasisVersion(redactionLog.getLegalBasisVersion())
+                .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
+                .build();
+    }
+}
@@ -1,14 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.awt.Color;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
+import com.iqser.red.service.redaction.v1.model.*;
+import lombok.RequiredArgsConstructor;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
@@ -21,15 +14,14 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.redaction.v1.model.CellRectangle;
-import com.iqser.red.service.redaction.v1.model.Comment;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.model.SectionRectangle;
-
-import lombok.RequiredArgsConstructor;
+import java.awt.Color;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;

@Service
@RequiredArgsConstructor
@@ -1,34 +1,24 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.awt.Color;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.SerializationUtils;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.configuration.v1.api.model.Colors;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResult;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
-
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.SerializationUtils;
+import org.springframework.stereotype.Service;
+
+import java.awt.Color;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static com.iqser.red.service.configuration.v1.api.resource.DictionaryResource.GLOBAL_DOSSIER;

@Slf4j
@Service
@@ -37,69 +27,89 @@ public class DictionaryService {

    private final DictionaryClient dictionaryClient;

-    private Map<String, DictionaryRepresentation> dictionariesByRuleSets = new HashMap<>();
+    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
+    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();


-    public long updateDictionary(String ruleSetId) {
+    public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {

-        long version = dictionaryClient.getVersion(ruleSetId);
-
-        var foundDictionary = dictionariesByRuleSets.get(ruleSetId);
-
-        if (foundDictionary == null || version > foundDictionary.getDictionaryVersion()) {
-            updateDictionaryEntry(ruleSetId, version);
+        long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER);
+        var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
+        if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
+            updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, GLOBAL_DOSSIER);
        }

-        return version;
+        long dossierDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, dossierId);
+        var dossierDictionary = dictionariesByDossier.get(dossierId);
+        if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
+            updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, dossierId);
+        }
+
+        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
    }


-    public DictionaryIncrement getDictionaryIncrements(String ruleSetId, long fromVersion) {
+    public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {

-        long version = updateDictionary(ruleSetId);
+        DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);

        Set<DictionaryIncrementValue> newValues = new HashSet<>();
-        List<DictionaryModel> dictionaryModels = dictionariesByRuleSets.get(ruleSetId).getDictionary();
+        List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
        dictionaryModels.forEach(dictionaryModel -> {
            dictionaryModel.getEntries().forEach(dictionaryEntry -> {
-                if (dictionaryEntry.getVersion() > fromVersion) {
+                if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
                }
            });
        });

+        if(dictionariesByDossier.containsKey(dossierId)) {
+            dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
+            dictionaryModels.forEach(dictionaryModel -> {
+                dictionaryModel.getEntries().forEach(dictionaryEntry -> {
+                    if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
+                        newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                    }
+                });
+            });
+        }
+
        return new DictionaryIncrement(newValues, version);
    }


-    private void updateDictionaryEntry(String ruleSetId, long version) {
+    private void updateDictionaryEntry(String dossierTemplateId, long version, String dossierId) {

        try {
            DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();

-            TypeResponse typeResponse = dictionaryClient.getAllTypes(ruleSetId);
+            TypeResponse typeResponse = dictionaryClient.getAllTypes(dossierTemplateId, dossierId);
            if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {

                List<DictionaryModel> dictionary = typeResponse.getTypes()
                        .stream()
                        .map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
-                                .isHint(), t.isRecommendation(), convertEntries(t), new HashSet<>()))
+                                .isHint(), t.isRecommendation(), convertEntries(t, dossierId), new HashSet<>(), !dossierId.equals(GLOBAL_DOSSIER)))
                        .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                        .collect(Collectors.toList());

                dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));

-                Colors colors = dictionaryClient.getColors(ruleSetId);
+                Colors colors = dictionaryClient.getColors(dossierTemplateId);

                dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
                dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
                dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
                dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
-                dictionaryRepresentation.setRuleSetId(ruleSetId);
+                dictionaryRepresentation.setDossierTemplateId(dossierTemplateId);
                dictionaryRepresentation.setDictionaryVersion(version);
                dictionaryRepresentation.setDictionary(dictionary);

-                dictionariesByRuleSets.put(ruleSetId, dictionaryRepresentation);
+                if(dossierId.equals(GLOBAL_DOSSIER)) {
+                    dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
+                } else {
+                    dictionariesByDossier.put(dossierId, dictionaryRepresentation);
+                }
            }
        } catch (FeignException e) {
            log.warn("Got some unknown feignException", e);
@@ -108,23 +118,23 @@ public class DictionaryService {
    }


-    public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) {
+    public void updateExternalDictionary(Dictionary dictionary, String dossierTemplateId) {

        dictionary.getDictionaryModels().forEach(dm -> {
            if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
-                dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false);
-                long externalVersion = dictionaryClient.getVersion(ruleSetId);
-                if (externalVersion == dictionary.getVersion() + 1) {
-                    dictionary.setVersion(externalVersion);
+                dictionaryClient.addEntries(dm.getType(), dossierTemplateId, new ArrayList<>(dm.getLocalEntries()), false, GLOBAL_DOSSIER);
+                long externalVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER);
+                if (externalVersion == dictionary.getVersion().getDossierTemplateVersion() + 1) {
+                    dictionary.getVersion().setDossierTemplateVersion(externalVersion);
                }
            }
        });
    }


-    private Set<DictionaryEntry> convertEntries(TypeResult t) {
+    private Set<DictionaryEntry> convertEntries(TypeResult t, String dossierId) {

-        Set<DictionaryEntry> entries = new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
+        Set<DictionaryEntry> entries = new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getDossierTemplateId(), dossierId)
                .getEntries());

        if (t.isCaseInsensitive()) {
@@ -141,9 +151,9 @@ public class DictionaryService {
    }


-    public boolean isCaseInsensitiveDictionary(String type, String ruleSetId) {
+    public boolean isCaseInsensitiveDictionary(String type, String dossierTemplateId) {

-        DictionaryModel dictionaryModel = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
+        DictionaryModel dictionaryModel = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
        if (dictionaryModel != null) {
            return dictionaryModel.isCaseInsensitive();
        }
@@ -151,19 +161,19 @@ public class DictionaryService {
    }


-    public float[] getColor(String type, String ruleSetId) {
+    public float[] getColor(String type, String dossierTemplateId) {

-        DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
+        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
        if (model != null) {
            return model.getColor();
        }
-        return dictionariesByRuleSets.get(ruleSetId).getDefaultColor();
+        return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
    }


-    public boolean isHint(String type, String ruleSetId) {
+    public boolean isHint(String type, String dossierTemplateId) {

-        DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
+        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
        if (model != null) {
            return model.isHint();
        }
@@ -171,9 +181,9 @@ public class DictionaryService {
    }


-    public boolean isRecommendation(String type, String ruleSetId) {
+    public boolean isRecommendation(String type, String dossierTemplateId) {

-        DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
+        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
        if (model != null) {
            return model.isRecommendation();
        }
@@ -181,35 +191,44 @@ public class DictionaryService {
    }


-    public Dictionary getDeepCopyDictionary(String ruleSetId) {
+    public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {

        List<DictionaryModel> copy = new ArrayList<>();

-        var representation = dictionariesByRuleSets.get(ruleSetId);
-        var dictionary = dictionariesByRuleSets.get(ruleSetId).getDictionary();
-        dictionary.forEach(dm -> {
+        var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
+        dossierTemplateRepresentation.getDictionary().forEach(dm -> {
            copy.add(SerializationUtils.clone(dm));
        });

-        return new Dictionary(copy, representation.getDictionaryVersion());
+        //TODO merge dictionaries if they have same names
+        long dossierDictionaryVersion = -1;
+        if(dictionariesByDossier.containsKey(dossierId)) {
+            var dossierRepresentation = dictionariesByDossier.get(dossierId);
+            dossierRepresentation.getDictionary().forEach(dm -> {
+                copy.add(SerializationUtils.clone(dm));
+            });
+            dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
+        }
+
+        return new Dictionary(copy, DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
    }


-    public float[] getRequestRemoveColor(String ruleSetId) {
+    public float[] getRequestRemoveColor(String dossierTemplateId) {

-        return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
+        return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
    }


-    public float[] getNotRedactedColor(String ruleSetId) {
+    public float[] getNotRedactedColor(String dossierTemplateId) {

-        return dictionariesByRuleSets.get(ruleSetId).getNotRedactedColor();
+        return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
    }


-    public float[] getRequestAddColor(String ruleSetId) {
+    public float[] getRequestAddColor(String dossierTemplateId) {

-        return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
+        return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
    }

-}
+}
@@ -1,11 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-
+import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import lombok.RequiredArgsConstructor;
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
@@ -15,12 +14,11 @@ import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-
-import lombok.RequiredArgsConstructor;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;

@Service
@RequiredArgsConstructor
@@ -28,16 +26,16 @@ public class DroolsExecutionService {

    private final RulesClient rulesClient;

-    private Map<String, KieContainer> kieContainers = new HashMap<>();
+    private final Map<String, KieContainer> kieContainers = new HashMap<>();

-    private Map<String, Long> rulesVersionPerRuleSetId = new HashMap<>();
+    private final Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();


-    public KieContainer getKieContainer(String ruleSetId) {
+    public KieContainer getKieContainer(String dossierTemplateId) {

-        KieContainer container = kieContainers.get(ruleSetId);
+        KieContainer container = kieContainers.get(dossierTemplateId);
        if (container == null) {
-            return createOrUpdateKieContainer(ruleSetId);
+            return createOrUpdateKieContainer(dossierTemplateId);
        } else {
            return container;
        }
@@ -57,43 +55,43 @@ public class DroolsExecutionService {
    }


-    public KieContainer updateRules(String ruleSetId) {
+    public KieContainer updateRules(String dossierTemplateId) {

-        long version = rulesClient.getVersion(ruleSetId);
-        Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId);
+        long version = rulesClient.getVersion(dossierTemplateId);
+        Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
        if (rulesVersion == null) {
            rulesVersion = -1L;
        }

        if (version > rulesVersion.longValue()) {
-            rulesVersionPerRuleSetId.put(ruleSetId, version);
-            return createOrUpdateKieContainer(ruleSetId);
+            rulesVersionPerDossierTemplateId.put(dossierTemplateId, version);
+            return createOrUpdateKieContainer(dossierTemplateId);
        }
-        return getKieContainer(ruleSetId);
+        return getKieContainer(dossierTemplateId);

    }


-    private KieContainer createOrUpdateKieContainer(String ruleSetId) {
+    private KieContainer createOrUpdateKieContainer(String dossierTemplateId) {

        try {

-            RulesResponse rules = rulesClient.getRules(ruleSetId);
+            RulesResponse rules = rulesClient.getRules(dossierTemplateId);
            if (rules == null || StringUtils.isEmpty(rules.getRules())) {
                throw new RuntimeException("Rules cannot be empty.");
            }

            KieServices kieServices = KieServices.Factory.get();
-            KieModule kieModule = getKieModule(ruleSetId, rules.getRules(), kieServices);
+            KieModule kieModule = getKieModule(dossierTemplateId, rules.getRules(), kieServices);

-            var container = kieContainers.get(ruleSetId);
+            var container = kieContainers.get(dossierTemplateId);
            if (container != null) {
                container.updateToVersion(kieModule.getReleaseId());
                return container;
            }

            container = kieServices.newKieContainer(kieModule.getReleaseId());
-            kieContainers.put(ruleSetId, container);
+            kieContainers.put(dossierTemplateId, container);
            return container;
        } catch (Exception e) {
            throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
@@ -102,11 +100,11 @@ public class DroolsExecutionService {
    }


-    private KieModule getKieModule(String ruleSetId, String rules, KieServices kieServices) {
+    private KieModule getKieModule(String dossierTemplateId, String rules, KieServices kieServices) {

        KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
        InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
-        kieFileSystem.write("src/main/resources/drools/rules" + ruleSetId + ".drl", kieServices.getResources()
+        kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources()
                .newInputStreamResource(input));
        KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
        kieBuilder.buildAll();
@@ -124,13 +122,13 @@ public class DroolsExecutionService {
    }


-    public long getRulesVersion(String ruleSetId) {
+    public long getRulesVersion(String dossierTemplateId) {

-        Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId);
+        Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
        if (rulesVersion == null) {
            return -1;
        }
        return rulesVersion.longValue();
    }

-}
+}
@@ -1,50 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
-import com.iqser.red.service.redaction.v1.server.classification.model.Header;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
+import com.iqser.red.service.redaction.v1.server.classification.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;

@Slf4j
@Service
@@ -56,13 +33,13 @@ public class EntityRedactionService {
    private final SurroundingWordsService surroundingWordsService;


-    public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
+    public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId) {

-        dictionaryService.updateDictionary(ruleSetId);
-        KieContainer container = droolsExecutionService.updateRules(ruleSetId);
-        long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId);
+        dictionaryService.updateDictionary(dossierTemplateId, dossierId);
+        KieContainer container = droolsExecutionService.updateRules(dossierTemplateId);
+        long rulesVersion = droolsExecutionService.getRulesVersion(dossierTemplateId);

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(dossierTemplateId, dossierId);

        Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));

@@ -94,11 +71,11 @@ public class EntityRedactionService {
                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
-                                .getStart(), entity.getEnd()));
+                                .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
            }
        }

-        dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
+        dictionaryService.updateExternalDictionary(dictionary, dossierTemplateId);

        classifiedDoc.setDictionaryVersion(dictionary.getVersion());
        classifiedDoc.setRulesVersion(rulesVersion);
@@ -210,6 +187,7 @@ public class EntityRedactionService {
                        .get(0)
                        .getPage());
                sectionText.getSectionAreas().add(sectionArea);
+                sectionText.getTextBlocks().addAll(cell.getTextBlocks());

                addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
                int cellStart = start;
@@ -258,6 +236,8 @@ public class EntityRedactionService {
                sectionText.setHeadline(table.getHeadline());
                sectionText.setSectionNumber(sectionNumber.intValue());
                sectionText.setTable(true);
+                sectionText.setTabularData(tabularData);
+                sectionText.setCellStarts(cellStarts);
                classifiedDoc.getSectionText().add(sectionText);
            }

@@ -290,6 +270,7 @@ public class EntityRedactionService {
                            .getSequences()
                            .get(0)
                            .getPage());
+                    sectionText.getTextBlocks().addAll(cell.getTextBlocks());
                    sectionText.getSectionAreas().add(sectionArea);
                }

@@ -348,6 +329,10 @@ public class EntityRedactionService {
            sectionText.setHeadline(headline);
            sectionText.setSectionNumber(sectionNumber.intValue());
            sectionText.setTable(false);
+            sectionText.setImages(images.stream()
+                    .map(image -> convert(image, sectionNumber.intValue(), headline))
+                    .collect(Collectors.toSet()));
+            sectionText.setTextBlocks(paragraphTextBlocks);
            classifiedDoc.getSectionText().add(sectionText);
        }

@@ -386,9 +371,9 @@ public class EntityRedactionService {
        String lowercaseInputString = searchableString.toLowerCase();
        for (DictionaryModel model : dictionary.getDictionaryModels()) {
            if (model.isCaseInsensitive()) {
-                found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary()));
            } else {
-                found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary()));
            }
        }

@@ -1,21 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import javax.imageio.ImageIO;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
 import com.iqser.red.service.redaction.v1.server.client.ImageClassificationResponse;
 import com.iqser.red.service.redaction.v1.server.client.MockMultipartFile;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import javax.imageio.ImageIO;
+import java.io.ByteArrayOutputStream;

@Slf4j
@Service
@@ -26,37 +23,49 @@ public class ImageClassificationService {
    private final RedactionServiceSettings settings;


-    public void classifyImages(Document classifiedDoc) {
+    public void classifyImages(Page page) {

-        long start = System.currentTimeMillis();
-        classifiedDoc.getPages().forEach(page -> {
-            page.getImages().forEach(image -> {
+        page.getImages().forEach(image -> {

-                if (settings.isEnableImageClassification()) {
-                    try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
-                        ImageIO.write(image.getImage(), "png", baos);
-                        ImageClassificationResponse response = imageClassificationClient.classify(new MockMultipartFile("file", "Image.png", "image/png", baos
-                                .toByteArray()));
-                        image.setImageType(ImageType.valueOf(response.getCategory()));
+            if (settings.isEnableImageClassification() && !isEntirePageImage(image, page)) {

-                    } catch (IOException e) {
-                        log.error("Could not classify image", e);
-                    }
-                } else {
+                long start = System.currentTimeMillis();
+                try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+                    ImageIO.write(image.getImage(), "png", baos);
+                    var mockFile = new MockMultipartFile("file", "Image.png", "image/png", baos.toByteArray());
+                    ImageClassificationResponse response = imageClassificationClient.classify(mockFile);
+                    image.setImageType(ImageType.valueOf(response.getCategory()));
+                } catch (Exception e) {
+                    log.error("Could not classify image", e);
                    image.setImageType(ImageType.OTHER);
                }
+                log.info("Image classification took: " + (System.currentTimeMillis() - start));
+            } else {
+                image.setImageType(ImageType.OTHER);
+            }

-                if (image.getImageType().equals(ImageType.OTHER)) {
-                    page.getTextBlocks().forEach(textblock -> {
-                        if (image.getPosition()
-                                .contains(textblock.getMinX(), textblock.getMinY(), textblock.getWidth(), textblock.getHeight())) {
-                            image.setImageType(ImageType.OCR);
-                        }
-                    });
-                }
-            });
+            image.getImage().flush();
+            image.setImage(null);
+
+            if (image.getImageType().equals(ImageType.OTHER)) {
+                page.getTextBlocks().forEach(textblock -> {
+                    if (image.getPosition()
+                            .contains(textblock.getMinX(), textblock.getMinY(), textblock.getWidth(), textblock.getHeight())) {
+                        image.setImageType(ImageType.OCR);
+                    }
+                });
+            }
        });
-        log.info("Image classification took: " + (System.currentTimeMillis() - start));
+
+    }
+
+    private boolean isEntirePageImage(PdfImage image, Page page){
+        double imageArea = image.getPosition().getHeight() * image.getPosition().getWidth();
+        if(imageArea / page.getCropBoxArea() >= settings.getMaxImageCropboxRatio()){
+            log.info("Skipping image classification because images is almost as large as the entire page");
+            return true;
+        }
+        return false;
    }

 }
@@ -1,53 +1,31 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.awt.geom.Rectangle2D;
-import java.io.ByteArrayInputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import com.iqser.red.service.file.management.v1.api.model.FileType;
+import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.Text;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;

-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;

-import com.iqser.red.service.redaction.v1.model.Comment;
-import com.iqser.red.service.redaction.v1.model.IdRemoval;
-import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
-import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
-import com.iqser.red.service.redaction.v1.model.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.ReanalyzeResult;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RenalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.parsing.PDFAreaTextStripper;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.ReanalysisSection;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
-
-import lombok.RequiredArgsConstructor;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;

+@Slf4j
@Service
@RequiredArgsConstructor
 public class ReanalyzeService {
@@ -57,39 +35,101 @@ public class ReanalyzeService {
    private final SurroundingWordsService surroundingWordsService;
    private final EntityRedactionService entityRedactionService;
    private final RedactionLogCreatorService redactionLogCreatorService;
+    private final RedactionStorageService redactionStorageService;
+    private final PdfSegmentationService pdfSegmentationService;
+    private final RedactionChangeLogService redactionChangeLogService;
+    private final AnalyzeResponseService analyzeResponseService;
+    private final LegalBasisClient legalBasisClient;
+
+    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+
+        var pageCount = 0;
+        Document classifiedDoc;
+
+        try {
+            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest
+                    .getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
+            classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
+            pageCount = classifiedDoc.getPages().size();
+        } catch (Exception e) {
+            throw new RedactionException(e);
+        }
+        log.info("Document structure analysis successful, starting redaction analysis...");
+
+        entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest
+                .getDossierId());
+        redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest
+                .getDossierTemplateId());
+
+        log.info("Redaction analysis successful...");
+
+        var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+        var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis,
+                classifiedDoc.getDictionaryVersion().getDossierTemplateVersion(),
+                classifiedDoc.getDictionaryVersion().getDossierVersion(),
+                classifiedDoc.getRulesVersion(),
+                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
+
+        log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc
+                .getDictionaryVersion(), analyzeRequest.getDossierTemplateId());
+
+        // first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data
+        var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
+        // store redactionLog
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc
+                .getSectionText()));
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
+                .getSectionGrid());
+
+        long duration = System.currentTimeMillis() - startTime;
+        return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog);
+    }


-    public ReanalyzeResult reanalyze(@RequestBody RenalyzeRequest renalyzeRequest) {
+    @SneakyThrows
+    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {

-        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(renalyzeRequest.getRuleSetId(), renalyzeRequest
-                .getRedactionLog()
-                .getDictionaryVersion());
+        long startTime = System.currentTimeMillis();

-        Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(renalyzeRequest.getManualRedactions());
+        var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+
+        // not yet ready for reanalysis
+        if (redactionLog == null || text == null || text.getNumberOfPages() == 0) {
+            return analyze(analyzeRequest);
+        }
+
+        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog
+                .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
+
+        Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions());
        Map<String, List<Comment>> comments = null;
        Set<ManualRedactionEntry> manualAdds = null;

-        if (renalyzeRequest.getManualRedactions() != null) {
+        if (analyzeRequest.getManualRedactions() != null) {
            // TODO comments will be removed from redactionLog, so we ignore this first.
-            comments = renalyzeRequest.getManualRedactions().getComments();
-            manualAdds = renalyzeRequest.getManualRedactions().getEntriesToAdd();
+            comments = analyzeRequest.getManualRedactions().getComments();
+            manualAdds = analyzeRequest.getManualRedactions().getEntriesToAdd();
        }

-        Set<Integer> sectionsToReanaylse = new HashSet<>();
+        Set<Integer> sectionsToReanalyse = new HashSet<>();
        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
-        for (RedactionLogEntry entry : renalyzeRequest.getRedactionLog().getRedactionLogEntry()) {
+        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
            if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) {
-                sectionsToReanaylse.add(entry.getSectionNumber());
+                sectionsToReanalyse.add(entry.getSectionNumber());
            }
            if (entry.isImage() || entry.getType().equals("image")) {
                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
            }
        }

-        for (SectionText sectionText : renalyzeRequest.getText().getSectionTexts()) {
+        for (SectionText sectionText : text.getSectionTexts()) {

            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrement.getValues())) {
-                sectionsToReanaylse.add(sectionText.getSectionNumber());
+                sectionsToReanalyse.add(sectionText.getSectionNumber());
            }

            if (manualAdds != null) {
@@ -106,187 +146,122 @@ public class ReanalyzeService {
            }
        }

-        if (sectionsToReanaylse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) {
-            renalyzeRequest.getRedactionLog().setDictionaryVersion(dictionaryIncrement.getDictionaryVersion());
-            return ReanalyzeResult.builder().redactionLog(renalyzeRequest.getRedactionLog()).build();
+        log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
+
+        if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) {
+            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
        }

-        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(renalyzeRequest.getDocument()))) {
+        List<SectionText> reanalysisSections = new ArrayList<>();

-            List<ReanalysisSection> reanalysisSections = new ArrayList<>();
-            for (SectionText sectionText : renalyzeRequest.getText().getSectionTexts()) {
+        for (SectionText sectionText : text.getSectionTexts()) {

-                if (!sectionsToReanaylse.contains(sectionText.getSectionNumber())) {
-                    continue;
-                }
-
-                ReanalysisSection reanalysisSection = new ReanalysisSection();
-                reanalysisSection.setHeadline(sectionText.getHeadline());
-                reanalysisSection.setSectionNumber(sectionText.getSectionNumber());
-                List<TextBlock> textBlocks = new ArrayList<>();
-
-                Map<Integer, List<SectionArea>> sectionAreasPerPage = new HashMap<>();
-                for (SectionArea sectionArea : sectionText.getSectionAreas()) {
-                    sectionAreasPerPage.computeIfAbsent(sectionArea.getPage(), (x) -> new ArrayList<>())
-                            .add(sectionArea);
-                }
-
-                Map<String, CellValue> tabularData = new HashMap<>();
-                List<Integer> cellStarts = new ArrayList<>();
-                for (Integer page : sectionAreasPerPage.keySet()) {
-                    List<SectionArea> areasOnPage = sectionAreasPerPage.get(page);
-
-                    PDPage pdPage = pdDocument.getPage(page - 1);
-                    PDRectangle cropBox = pdPage.getCropBox();
-                    PDFAreaTextStripper textStripper = new PDFAreaTextStripper();
-                    textStripper.setPageNumber(page);
-
-                    int cellStart = 0;
-                    for (SectionArea sectionArea : areasOnPage) {
-
-                        Rectangle2D rect = null;
-                        if (pdPage.getRotation() == 90) {
-                            rect = new Rectangle2D.Float(sectionArea.getTopLeft().getY(), sectionArea.getTopLeft()
-                                    .getX(), sectionArea.getHeight(), sectionArea.getWidth() + 0.001f);
-                        } else {
-                            rect = new Rectangle2D.Float(sectionArea.getTopLeft().getX(), -sectionArea.getTopLeft()
-                                    .getY() + cropBox.getUpperRightY() - sectionArea.getHeight(), sectionArea.getWidth(), sectionArea
-                                    .getHeight() + 0.001f);
-                        }
-
-                        textStripper.addRegion(String.valueOf(1), rect);
-                        textStripper.extractRegions(pdPage);
-                        textStripper.getTextForRegion(String.valueOf(1));
-                        List<TextPositionSequence> positions = textStripper.getTextPositionSequences();
-
-                        TextBlock textBlock = new TextBlock(sectionArea.getTopLeft().getX(), sectionArea.getTopLeft()
-                                .getX() + sectionArea.getWidth(), sectionArea.getTopLeft()
-                                .getY(), sectionArea.getTopLeft().getY() + sectionArea.getHeight(), positions, 0);
-
-                        if (sectionText.isTable()) {
-                            Cell cell = new Cell();
-                            cell.addTextBlock(textBlock);
-                            tabularData.put(sectionArea.getHeader(), new CellValue(cell.getTextBlocks(), cellStart));
-                            cellStarts.add(cellStart);
-                            cellStart = cellStart + cell.toString().trim().length() + 1;
-                        }
-
-                        textBlocks.add(textBlock);
-                        textStripper.clearPositions();
-                    }
-
-                }
-                reanalysisSection.setTextBlocks(textBlocks);
-                reanalysisSection.setTabularData(tabularData);
-
-                if (sectionText.isTable()) {
-                    reanalysisSection.setCellStarts(cellStarts);
-                }
-                if (imageEntries.containsKey(sectionText.getSectionNumber())) {
-                    reanalysisSection.getImages().addAll(imageEntries.get(sectionText.getSectionNumber()));
-                }
-
-                reanalysisSections.add(reanalysisSection);
+            if (sectionsToReanalyse.contains(sectionText.getSectionNumber())) {
+                reanalysisSections.add(sectionText);
            }
-
-            //--
-
-            KieContainer kieContainer = droolsExecutionService.updateRules(renalyzeRequest.getRuleSetId());
-
-            Dictionary dictionary = dictionaryService.getDeepCopyDictionary(renalyzeRequest.getRuleSetId());
-
-            List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
-            for (ReanalysisSection reanalysisSection : reanalysisSections) {
-
-                Set<Entity> entities = entityRedactionService.findEntities(reanalysisSection.getSearchableText(), reanalysisSection
-                        .getHeadline(), reanalysisSection.getSectionNumber(), dictionary, false);
-                if (reanalysisSection.getCellStarts() != null) {
-                    surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection
-                            .getCellStarts());
-                } else {
-                    surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
-                }
-
-                sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
-                        .isLocal(false)
-                        .dictionaryTypes(dictionary.getTypes())
-                        .entities(entities)
-                        .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
-                        .searchText(reanalysisSection.getSearchableText().toString())
-                        .headline(reanalysisSection.getHeadline())
-                        .sectionNumber(reanalysisSection.getSectionNumber())
-                        .tabularData(reanalysisSection.getTabularData())
-                        .searchableText(reanalysisSection.getSearchableText())
-                        .dictionary(dictionary)
-                        .images(reanalysisSection.getImages())
-                        .build(), reanalysisSection.getSearchableText()));
-            }
-
-            Set<Entity> entities = new HashSet<>();
-            Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
-            sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
-                Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair
-                        .getSection());
-                entities.addAll(analysedRowSection.getEntities());
-                EntitySearchUtils.removeEntitiesContainedInLarger(entities);
-
-                for (Image image : analysedRowSection.getImages()) {
-                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
-                }
-
-            });
-
-            Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
-            for (Entity entity : entities) {
-                Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
-                for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
-                    sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
-                            .add(entityPositionSequence);
-                }
-
-                for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
-                    entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
-                            .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
-                                    .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
-                                    .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
-                                    .getStart(), entity.getEnd()));
-                }
-            }
-
-            List<RedactionLogEntry> newRedactionLogEntries = new ArrayList<>();
-            for (int page = 1; page <= pdDocument.getNumberOfPages(); page++) {
-                if (entitiesPerPage.get(page) != null) {
-                    newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, renalyzeRequest
-                            .getManualRedactions(), page, renalyzeRequest.getRuleSetId()));
-                }
-
-                if (imagesPerPage.get(page) != null) {
-                    newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, renalyzeRequest
-                            .getManualRedactions(), page, renalyzeRequest.getRuleSetId()));
-                }
-
-                newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, renalyzeRequest
-                        .getRuleSetId()));
-            }
-
-            Iterator<RedactionLogEntry> itty = renalyzeRequest.getRedactionLog().getRedactionLogEntry().iterator();
-            while (itty.hasNext()) {
-                RedactionLogEntry entry = itty.next();
-                if (sectionsToReanaylse.contains(entry.getSectionNumber())) {
-                    itty.remove();
-                }
-            }
-
-            renalyzeRequest.getRedactionLog().getRedactionLogEntry().addAll(newRedactionLogEntries);
-
-            renalyzeRequest.getRedactionLog().setDictionaryVersion(dictionaryIncrement.getDictionaryVersion());
-
-            return ReanalyzeResult.builder().redactionLog(renalyzeRequest.getRedactionLog()).build();
-
-        } catch (Exception e) {
-            throw new RedactionException(e);
        }

+        //--
+
+        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
+
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
+        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
+        for (SectionText reanalysisSection : reanalysisSections) {
+
+            Set<Entity> entities = entityRedactionService.findEntities(reanalysisSection.getSearchableText(), reanalysisSection
+                    .getHeadline(), reanalysisSection.getSectionNumber(), dictionary, false);
+            if (reanalysisSection.getCellStarts() != null) {
+                surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection
+                        .getCellStarts());
+            } else {
+                surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
+            }
+
+            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
+                    .isLocal(false)
+                    .dictionaryTypes(dictionary.getTypes())
+                    .entities(entities)
+                    .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
+                    .searchText(reanalysisSection.getSearchableText().toString())
+                    .headline(reanalysisSection.getHeadline())
+                    .sectionNumber(reanalysisSection.getSectionNumber())
+                    .tabularData(reanalysisSection.getTabularData())
+                    .searchableText(reanalysisSection.getSearchableText())
+                    .dictionary(dictionary)
+                    .images(reanalysisSection.getImages())
+                    .build(), reanalysisSection.getSearchableText()));
+        }
+
+        Set<Entity> entities = new HashSet<>();
+        Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
+        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
+            Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
+            entities.addAll(analysedRowSection.getEntities());
+            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+
+            for (Image image : analysedRowSection.getImages()) {
+                imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
+            }
+
+        });
+
+        Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
+        for (Entity entity : entities) {
+            Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
+            for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
+                sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
+                        .add(entityPositionSequence);
+            }
+
+            for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
+                entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
+                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
+                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
+                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
+                                .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
+            }
+        }
+
+        List<RedactionLogEntry> newRedactionLogEntries = new ArrayList<>();
+        for (int page = 1; page <= text.getNumberOfPages(); page++) {
+            if (entitiesPerPage.get(page) != null) {
+                newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
+                        .getDossierTemplateId()));
+            }
+
+            if (imagesPerPage.get(page) != null) {
+                newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
+                        .getDossierTemplateId()));
+            }
+
+            newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest
+                    .getDossierTemplateId()));
+        }
+
+        redactionLog.getRedactionLogEntry()
+                .removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
+        redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
+
+    }
+
+
+    private AnalyzeResult finalizeAnalysis(@RequestBody AnalyzeRequest analyzeRequest, long startTime,
+                                           RedactionLog redactionLog, Text text,
+                                           DictionaryIncrement dictionaryIncrement) {
+
+        redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierTemplateVersion());
+        redactionLog.setDossierDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierVersion());
+
+        var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
+
+        long duration = System.currentTimeMillis() - startTime;
+
+        return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text
+                .getNumberOfPages(), redactionLog, changeLog);
    }


@@ -309,7 +284,7 @@ public class ReanalyzeService {

        return Image.builder()
                .type(entry.getType())
-                .position(new Rectangle2D.Float(position.getTopLeft().getX(), position.getTopLeft()
+                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft()
                        .getY(), position.getWidth(), position.getHeight()))
                .sectionNumber(entry.getSectionNumber())
                .section(entry.getSection())
@@ -0,0 +1,96 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import com.iqser.red.service.file.management.v1.api.model.FileType;
+import com.iqser.red.service.redaction.v1.model.ChangeType;
+import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
+import com.iqser.red.service.redaction.v1.model.RedactionChangeLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class RedactionChangeLogService {
+
+    private final RedactionStorageService redactionStorageService;
+
+    public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) {
+
+        try {
+            RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId);
+            var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog);
+            redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog);
+            return changeLog;
+        } catch (Exception e) {
+            log.debug("Previous redaction log not available");
+            return null;
+        }
+
+    }
+
+
+    private RedactionChangeLog createChangeLog(RedactionLog currentRedactionLog, RedactionLog previousRedactionLog) {
+
+
+        if (previousRedactionLog == null) {
+            return null;
+        }
+
+        List<RedactionLogEntry> added = new ArrayList<>(currentRedactionLog.getRedactionLogEntry());
+        added.removeAll(previousRedactionLog.getRedactionLogEntry());
+
+        List<RedactionLogEntry> removed = new ArrayList<>(previousRedactionLog.getRedactionLogEntry());
+        removed.removeAll(currentRedactionLog.getRedactionLogEntry());
+
+        List<RedactionChangeLogEntry> changeLogEntries = added.stream()
+                .map(entry -> convert(entry, ChangeType.ADDED))
+                .collect(Collectors.toList());
+        changeLogEntries.addAll(removed.stream()
+                .map(entry -> convert(entry, ChangeType.REMOVED))
+                .collect(Collectors.toList()));
+
+        return new RedactionChangeLog(changeLogEntries,
+                currentRedactionLog.getDictionaryVersion(),
+                currentRedactionLog.getDossierDictionaryVersion(),
+                currentRedactionLog.getRulesVersion(),
+                currentRedactionLog.getLegalBasisVersion());
+    }
+
+
+    private RedactionChangeLogEntry convert(RedactionLogEntry entry, ChangeType changeType) {
+
+        return RedactionChangeLogEntry.builder()
+                .id(entry.getId())
+                .type(entry.getType())
+                .value(entry.getValue())
+                .reason(entry.getReason())
+                .matchedRule(entry.getMatchedRule())
+                .legalBasis(entry.getLegalBasis())
+                .redacted(entry.isRedacted())
+                .isHint(entry.isHint())
+                .isRecommendation(entry.isRecommendation())
+                .section(entry.getSection())
+                .color(entry.getColor())
+                .positions(entry.getPositions())
+                .sectionNumber(entry.getSectionNumber())
+                .manual(entry.isManual())
+                .status(entry.getStatus())
+                .manualRedactionType(entry.getManualRedactionType())
+                .isDictionaryEntry(entry.isDictionaryEntry())
+                .textBefore(entry.getTextBefore())
+                .textAfter(entry.getTextAfter())
+                .comments(entry.getComments())
+                .changeType(changeType)
+                .isDossierDictionaryEntry(entry.isDossierDictionaryEntry())
+                .build();
+    }
+
+}
@@ -1,31 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.pdfbox.text.TextPosition;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.model.CellRectangle;
-import com.iqser.red.service.redaction.v1.model.Comment;
-import com.iqser.red.service.redaction.v1.model.IdRemoval;
-import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
-import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
-import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
-import com.iqser.red.service.redaction.v1.model.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.SectionRectangle;
-import com.iqser.red.service.redaction.v1.model.Status;
+import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@@ -34,8 +13,16 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.RequiredArgsConstructor;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;

@Service
@RequiredArgsConstructor
@@ -45,7 +32,7 @@ public class RedactionLogCreatorService {


    public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
-                                   String ruleSetId) {
+                                   String dossierTemplateId) {

        Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);

@@ -55,24 +42,24 @@ public class RedactionLogCreatorService {

            if (classifiedDoc.getEntities().get(page) != null) {
                classifiedDoc.getRedactionLogEntities()
-                        .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, ruleSetId));
+                        .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId));
            }

            if (manualRedactionPages.contains(page)) {
                classifiedDoc.getRedactionLogEntities()
-                        .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, ruleSetId));
+                        .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId));
            }

            if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) {
                classifiedDoc.getRedactionLogEntities()
-                        .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, ruleSetId));
+                        .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId));
            }
        }
    }


    public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, ManualRedactions manualRedactions,
-                                                   int pageNumber, String ruleSetId) {
+                                                   int pageNumber, String dossierTemplateId) {

        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

@@ -82,14 +69,14 @@ public class RedactionLogCreatorService {

            RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
                    .id(id)
-                    .color(getColorForImage(image, ruleSetId, false))
+                    .color(getColorForImage(image, dossierTemplateId, false))
                    .isImage(true)
                    .type(image.getType())
                    .redacted(image.isRedaction())
                    .reason(image.getRedactionReason())
                    .legalBasis(image.getLegalBasis())
                    .matchedRule(image.getMatchedRule())
-                    .isHint(dictionaryService.isHint(image.getType(), ruleSetId))
+                    .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId))
                    .manual(false)
                    .isDictionaryEntry(false)
                    .isRecommendation(false)
@@ -109,11 +96,11 @@ public class RedactionLogCreatorService {
                            redactionLogEntry.setRedacted(false);
                            redactionLogEntry.setStatus(Status.APPROVED);
                            manualOverrideReason = image.getRedactionReason() + ", removed by manual override";
-                            redactionLogEntry.setColor(getColorForImage(image, ruleSetId, false));
+                            redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false));
                        } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
                            manualOverrideReason = image.getRedactionReason() + ", requested to remove";
                            redactionLogEntry.setStatus(Status.REQUESTED);
-                            redactionLogEntry.setColor(getColorForImage(image, ruleSetId, true));
+                            redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true));
                        } else {
                            redactionLogEntry.setStatus(Status.DECLINED);
                        }
@@ -134,13 +121,13 @@ public class RedactionLogCreatorService {
                            image.setRedaction(true);
                            redactionLogEntry.setRedacted(true);
                            redactionLogEntry.setStatus(Status.APPROVED);
-                            redactionLogEntry.setColor(getColorForImage(image, ruleSetId, false));
+                            redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false));
                            manualOverrideReason = image.getRedactionReason() + ", forced by manual override";
                            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                        } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
                            manualOverrideReason = image.getRedactionReason() + ", requested to force redact";
                            redactionLogEntry.setStatus(Status.REQUESTED);
-                            redactionLogEntry.setColor(getColorForImage(image, ruleSetId, true));
+                            redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true));
                            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                        } else {
                            redactionLogEntry.setStatus(Status.DECLINED);
@@ -179,7 +166,7 @@ public class RedactionLogCreatorService {


    public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions,
-                                              int page, String ruleSetId) {
+                                              int page, String dossierTemplateId) {

        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

@@ -193,7 +180,7 @@ public class RedactionLogCreatorService {

            for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {

-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, ruleSetId);
+                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId);
                if (processedIds.contains(entityPositionSequence.getId())) {

                    // TODO refactor this outer loop jump as soon as we have the time.
@@ -212,11 +199,11 @@ public class RedactionLogCreatorService {
                                redactionLogEntry.setRedacted(false);
                                redactionLogEntry.setStatus(Status.APPROVED);
                                manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
-                                redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
+                                redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false));
                            } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
                                manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
                                redactionLogEntry.setStatus(Status.REQUESTED);
-                                redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
+                                redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true));
                            } else {
                                redactionLogEntry.setStatus(Status.DECLINED);
                            }
@@ -237,13 +224,13 @@ public class RedactionLogCreatorService {
                                entity.setRedaction(true);
                                redactionLogEntry.setRedacted(true);
                                redactionLogEntry.setStatus(Status.APPROVED);
-                                redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
+                                redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false));
                                manualOverrideReason = entity.getRedactionReason() + ", forced by manual override";
                                redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                            } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
                                manualOverrideReason = entity.getRedactionReason() + ", requested to force redact";
                                redactionLogEntry.setStatus(Status.REQUESTED);
-                                redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
+                                redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true));
                                redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                            } else {
                                redactionLogEntry.setStatus(Status.DECLINED);
@@ -285,24 +272,24 @@ public class RedactionLogCreatorService {
    }


-    private List<Rectangle> getRectanglesPerLine(List<TextPosition> textPositions, int page) {
+    private List<Rectangle> getRectanglesPerLine(List<RedTextPosition> textPositions, int page) {

        List<Rectangle> rectangles = new ArrayList<>();
        if (textPositions.size() == 1) {
-            rectangles.add(new TextPositionSequence(textPositions, page).getRectangle());
+            rectangles.add( TextPositionSequence.fromData(textPositions, page).getRectangle());
        } else {
            float y = textPositions.get(0).getYDirAdj();
            int startIndex = 0;
            for (int i = 1; i < textPositions.size(); i++) {
                float yDirAdj = textPositions.get(i).getYDirAdj();
                if (yDirAdj != y) {
-                    rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, i), page).getRectangle());
+                    rectangles.add( TextPositionSequence.fromData(textPositions.subList(startIndex, i), page).getRectangle());
                    y = yDirAdj;
                    startIndex = i;
                }
            }
            if (startIndex != textPositions.size()) {
-                rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
+                rectangles.add( TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
            }
        }

@@ -312,7 +299,7 @@ public class RedactionLogCreatorService {

    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
                                                       Map<String, List<Comment>> comments, int page,
-                                                       String ruleSetId) {
+                                                       String dossierTemplateId) {

        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

@@ -324,7 +311,7 @@ public class RedactionLogCreatorService {

            String id = manualRedactionEntry.getId();

-            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, ruleSetId);
+            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId);

            List<Rectangle> rectanglesOnPage = new ArrayList<>();
            for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
@@ -351,11 +338,11 @@ public class RedactionLogCreatorService {


    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
-                                                      String ruleSetId) {
+                                                      String dossierTemplateId) {

        return RedactionLogEntry.builder()
                .id(id)
-                .color(getColorForManualAdd(manualRedactionEntry.getType(), ruleSetId, manualRedactionEntry.getStatus()))
+                .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
                .reason(manualRedactionEntry.getReason())
                .legalBasis(manualRedactionEntry.getLegalBasis())
                .value(manualRedactionEntry.getValue())
@@ -368,21 +355,22 @@ public class RedactionLogCreatorService {
                .status(manualRedactionEntry.getStatus())
                .manualRedactionType(ManualRedactionType.ADD)
                .isDictionaryEntry(false)
+                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
                .build();
    }


-    private RedactionLogEntry createRedactionLogEntry(Entity entity, String ruleSetId) {
+    private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) {

        return RedactionLogEntry.builder()
-                .color(getColor(entity, ruleSetId, false))
+                .color(getColor(entity, dossierTemplateId, false))
                .reason(entity.getRedactionReason())
                .legalBasis(entity.getLegalBasis())
                .value(entity.getWord())
                .type(entity.getType())
                .redacted(entity.isRedaction())
-                .isHint(isHint(entity, ruleSetId))
-                .isRecommendation(isRecommendation(entity, ruleSetId))
+                .isHint(isHint(entity, dossierTemplateId))
+                .isRecommendation(isRecommendation(entity, dossierTemplateId))
                .section(entity.getHeadline())
                .sectionNumber(entity.getSectionNumber())
                .matchedRule(entity.getMatchedRule())
@@ -391,60 +379,61 @@ public class RedactionLogCreatorService {
                .textBefore(entity.getTextBefore())
                .startOffset(entity.getStart())
                .endOffset(entity.getEnd())
+                .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
                .build();
    }


-    private float[] getColor(Entity entity, String ruleSetId, boolean requestedToRemove) {
+    private float[] getColor(Entity entity, String dossierTemplateId, boolean requestedToRemove) {

        if (requestedToRemove) {
-            return dictionaryService.getRequestRemoveColor(ruleSetId);
+            return dictionaryService.getRequestRemoveColor(dossierTemplateId);
        }
-        if (!entity.isRedaction() && !isHint(entity, ruleSetId)) {
-            return dictionaryService.getNotRedactedColor(ruleSetId);
+        if (!entity.isRedaction() && !isHint(entity, dossierTemplateId)) {
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
        }
-        return dictionaryService.getColor(entity.getType(), ruleSetId);
+        return dictionaryService.getColor(entity.getType(), dossierTemplateId);
    }


-    private float[] getColorForManualAdd(String type, String ruleSetId, Status status) {
+    private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) {

        if (status.equals(Status.REQUESTED)) {
-            return dictionaryService.getRequestAddColor(ruleSetId);
+            return dictionaryService.getRequestAddColor(dossierTemplateId);
        } else if (status.equals(Status.DECLINED)) {
-            return dictionaryService.getNotRedactedColor(ruleSetId);
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
        }
-        return getColor(type, ruleSetId);
+        return getColor(type, dossierTemplateId);
    }


-    private float[] getColor(String type, String ruleSetId) {
+    private float[] getColor(String type, String dossierTemplateId) {

-        return dictionaryService.getColor(type, ruleSetId);
+        return dictionaryService.getColor(type, dossierTemplateId);
    }


-    private float[] getColorForImage(Image image, String ruleSetId, boolean requestedToRemove) {
+    private float[] getColorForImage(Image image, String dossierTemplateId, boolean requestedToRemove) {

        if (requestedToRemove) {
-            return dictionaryService.getRequestRemoveColor(ruleSetId);
+            return dictionaryService.getRequestRemoveColor(dossierTemplateId);
        }
-        if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), ruleSetId)) {
-            return dictionaryService.getNotRedactedColor(ruleSetId);
+        if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), dossierTemplateId)) {
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
        }
-        return dictionaryService.getColor(image.getType(), ruleSetId);
+        return dictionaryService.getColor(image.getType(), dossierTemplateId);
    }


-    private boolean isHint(Entity entity, String ruleSetId) {
+    private boolean isHint(Entity entity, String dossierTemplateId) {

-        return dictionaryService.isHint(entity.getType(), ruleSetId);
+        return dictionaryService.isHint(entity.getType(), dossierTemplateId);
    }


-    private boolean isRecommendation(Entity entity, String ruleSetId) {
+    private boolean isRecommendation(Entity entity, String dossierTemplateId) {

-        return dictionaryService.isRecommendation(entity.getType(), ruleSetId);
+        return dictionaryService.isRecommendation(entity.getType(), dossierTemplateId);
    }


@@ -1,25 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
@@ -46,7 +38,7 @@ public class EntitySearchUtils {

                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    if(value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())){
+                    if (value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())) {
                        return true;
                    }
                }
@@ -57,7 +49,7 @@ public class EntitySearchUtils {


    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                            boolean local) {
+                            boolean local, boolean isDossierDictionary) {

        Set<Entity> found = new HashSet<>();

@@ -77,7 +69,7 @@ public class EntitySearchUtils {

                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local));
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local, isDossierDictionary));
                }
            } while (startIndex > -1);
        }
@@ -147,16 +139,16 @@ public class EntitySearchUtils {

    public void addEntitiesWithHigherRank(Set<Entity> entities, Entity found, Dictionary dictionary) {

-        if(entities.contains(found)){
+        if (entities.contains(found)) {
            Entity existing = entities.stream().filter(entity -> entity.equals(found)).findFirst().get();
-            if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())){
+            if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
                entities.remove(found);
            }
        }
        entities.add(found);
    }

-    public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found){
+    public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found) {
        // HashSet keeps old value but we want the new.
        entities.removeAll(found);
        entities.addAll(found);
@@ -1,15 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import java.awt.geom.Rectangle2D;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
 import lombok.experimental.UtilityClass;

+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
@UtilityClass
 public class IdBuilder {

@@ -26,14 +25,9 @@ public class IdBuilder {
    }


-    public String buildId(Rectangle2D rectangle2D, int page){
-
-        StringBuilder sb = new StringBuilder();
-        sb.append("x").append(rectangle2D.getX()).append("y").append(rectangle2D.getY()).append("h").append(rectangle2D.getHeight()).append("w").append(rectangle2D.getWidth()).append("p").append(page);
-
-        return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
+    public String buildId(RedRectangle2D rectangle2D, int page) {
+        return hashFunction.hashString("x" + rectangle2D.getX() + "y" + rectangle2D.getY() + "h" + rectangle2D.getHeight() + "w" + rectangle2D.getWidth() + "p" + page, StandardCharsets.UTF_8).toString();
    }


-
 }
@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import lombok.experimental.UtilityClass;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -8,8 +10,6 @@ import java.nio.charset.StandardCharsets;
 import java.util.Set;
 import java.util.stream.Collectors;

-import lombok.experimental.UtilityClass;
-
@UtilityClass
 public class ResourceLoader {

@@ -27,4 +27,4 @@ public class ResourceLoader {

    }

-}
+}
@@ -7,6 +7,7 @@ public class TextNormalizationUtilities {

    /**
     * Revert hyphenation due to line breaks.
+     *
     * @param text Text to be processed.
     * @return Text without line-break hyphenation.
     */
@@ -14,4 +15,4 @@ public class TextNormalizationUtilities {
        return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2");
    }

-}
+}
@@ -1,88 +1,291 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;

-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
 import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
+import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
 import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ImageClassificationService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.pdfbox.io.MemoryUsageSetting;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.springframework.stereotype.Service;
+
+import java.awt.Graphics;
+import java.awt.geom.Rectangle2D;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;

@Slf4j
@Service
@RequiredArgsConstructor
 public class PdfSegmentationService {

+    private final static int MAX_PAGES_BEFORE_GC = 250;
+
    private final RulingCleaningService rulingCleaningService;
    private final TableExtractionService tableExtractionService;
    private final BlockificationService blockificationService;
    private final ClassificationService classificationService;
    private final SectionsBuilderService sectionsBuilderService;
+    private final ImageClassificationService imageClassificationService;


-    public Document parseDocument(PDDocument pdDocument) throws IOException {
+    public Document parseDocument(InputStream documentInputStream) throws IOException {
+        return parseDocument(documentInputStream, false);
+    }

-        Document document = new Document();
+    public Document parseDocument(InputStream documentInputStream, boolean ignoreImages) throws IOException {
+        PDDocument pdDocument = null;
+        try {
+            //create tempFile
+            File tempFile = File.createTempFile("document", ".pdf");
+            IOUtils.copy(documentInputStream, new FileOutputStream(tempFile));

-        List<Page> pages = new ArrayList<>();
-        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-        for (int pageNumber = 1; pageNumber <= pdDocument.getNumberOfPages(); pageNumber++) {
-            PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-            stripper.setPageNumber(pageNumber);
-            stripper.setStartPage(pageNumber);
-            stripper.setEndPage(pageNumber);
-            stripper.setPdpage(pdPage);
-            stripper.getText(pdDocument);
+            // initialize required variables
+            Document document = new Document();
+            List<Page> pages = new ArrayList<>();

-            PDRectangle pdr = pdPage.getMediaBox();
-            boolean isLandscape = pdr.getWidth() > pdr.getHeight();

-            int rotation = pdPage.getRotation();
-            boolean isRotated = rotation != 0 && rotation != 360;
+            pdDocument = reinitializePDDocument(tempFile, null);
+            long pageCount = pdDocument.getNumberOfPages();

-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
-                    .getMaxCharHeight());
+            for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {

-            Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
-                    .getVertical());
-            page.setRotation(rotation);
+                if (pageNumber % MAX_PAGES_BEFORE_GC == 0) {
+                    pdDocument = reinitializePDDocument(tempFile, pdDocument);
+                }

-            tableExtractionService.extractTables(cleanRulings, page);
+                PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+                PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+                stripper.setPageNumber(pageNumber);
+                stripper.setStartPage(pageNumber);
+                stripper.setEndPage(pageNumber);
+                stripper.setPdpage(pdPage);
+                stripper.getText(pdDocument);

-            buildPageStatistics(page);
+                PDRectangle pdr = pdPage.getMediaBox();
+                boolean isLandscape = pdr.getWidth() > pdr.getHeight();

-            page.setLandscape(isLandscape || isRotated);
+                int rotation = pdPage.getRotation();
+                boolean isRotated = rotation != 0 && rotation != 360;

-            page.setPageNumber(pageNumber);
-            increaseDocumentStatistics(page, document);
+                CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
+                        .getMaxCharHeight());

-            page.setImages(stripper.getImages());
-            pages.add(page);
+                Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
+                        .getVertical());
+
+                PDRectangle cropbox = pdPage.getCropBox();
+                float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
+                page.setCropBoxArea(cropboxArea);
+
+                page.setRotation(rotation);
+                page.setLandscape(isLandscape || isRotated);
+                page.setPageNumber(pageNumber);
+                List<PdfImage> mergedList = processImages(stripper.getImages());
+
+                List<PdfImage> imagesInImage = new ArrayList<>();
+                for(PdfImage image: mergedList){
+                    for (PdfImage inner: mergedList){
+                        if(image != inner && image.getPosition().contains(inner.getPosition().getX(), inner.getPosition().getY(), inner.getPosition().getWidth(), inner.getPosition().getHeight())){
+                            imagesInImage.add(inner);
+                        }
+                    }
+                }
+                mergedList.removeAll(imagesInImage);
+
+                page.setImages(mergedList);
+
+                tableExtractionService.extractTables(cleanRulings, page);
+                buildPageStatistics(page);
+                increaseDocumentStatistics(page, document);
+
+
+                if (!ignoreImages) {
+                    imageClassificationService.classifyImages(page);
+                }
+
+                pages.add(page);
+
+
+            }
+
+            document.setPages(pages);
+
+            classificationService.classifyDocument(document);
+            sectionsBuilderService.buildSections(document);
+            sectionsBuilderService.addImagesToSections(document);
+
+            pdDocument = reinitializePDDocument(tempFile, pdDocument);
+
+            IOUtils.close(pdDocument);
+
+            tempFile.delete();
+
+            return document;
+        } finally {
+            if (pdDocument != null) {
+                pdDocument.close();
+            }
+        }
+    }
+
+    private PDDocument reinitializePDDocument(File tempFile, PDDocument pdDocument) throws IOException {
+        if (pdDocument != null) {
+            pdDocument.close();
+        }
+        System.runFinalization();
+        System.gc();
+
+        MemoryStats.printMemoryStats();
+
+        var newPDDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupTempFileOnly());
+        newPDDocument.setAllSecurityToBeRemoved(true);
+
+        return newPDDocument;
+    }
+
+    //merge images, if they are separated during pdf import, return new list of Pdfimages
+    private List<PdfImage> processImages(List<PdfImage> imageList) {
+        if (imageList.size() > 1) {
+            List<PdfImage> mergedList = new ArrayList<>();
+            int countElementsInList = 0;
+            boolean beginImage = true;
+
+            // a List of Boolean, true = candidate for merging, false = no merging
+            List<Boolean> candidatesList = getCandidatesList(imageList);
+
+            // loop through list, if there are candidates for merging (true), merge images and add it to mergedList
+            for (int i = 0; i < candidatesList.size(); i++) {
+                if (candidatesList.get(i)) {
+                    if (beginImage) {
+                        //begin of image, merge two parts of imageList
+                        PdfImage mergedImage = mergeTwoImages(imageList.get(i), imageList.get(i + 1));
+                        // image merge successful
+                        if (mergedImage != null) {
+                            mergedList.add(mergedImage);
+                            countElementsInList++;
+                        }
+                    } else {
+                        //middle of an image, merge current piece auf mergedList with image of imageList
+                        PdfImage mergedImage = mergeTwoImages(mergedList.get(countElementsInList - 1), imageList.get(i + 1));
+                        // image merge successful
+                        if (mergedImage != null) {
+                            mergedList.set(countElementsInList - 1, mergedImage);
+                        }
+                    }
+                    beginImage = false;
+                } else {
+                    // if the last candidate is false, then both images i and i+1 must be added
+                    if (i == candidatesList.size() - 1) {
+                        if (countElementsInList > 0 && mergedList.get(countElementsInList - 1) == imageList.get(i)) {
+                            mergedList.add(imageList.get(i + 1));
+                        } else {
+                            mergedList.add(imageList.get(i));
+                            mergedList.add(imageList.get(i + 1));
+                        }
+                    } else {
+                        //first image is not splitted, add i to resultlist
+                        if (beginImage) {
+                            mergedList.add(imageList.get(i));
+                            countElementsInList++;
+                        } else {
+                            // i is the end of an image, add begin of new image
+                            mergedList.add(imageList.get(i + 1));
+                            countElementsInList++;
+                            beginImage = false;
+                        }
+                    }
+                }
+            }
+            return mergedList;
+        } else {
+            return imageList;
+        }
+    }
+
+    private PdfImage mergeTwoImages(PdfImage image1, PdfImage image2) {
+
+        // diese Angaben von getPosition scheinen nicht richtig zu sein, damit werden teile des Bildes abgeschnitten
+        double width = image1.getPosition().getWidth();
+        double height1 = image1.getPosition().getHeight();
+        double height2 = image2.getPosition().getHeight();
+        // mit  den Werten, die unter Image gespeichert sind, funktioniert es
+        double img1height = image1.getImage().getHeight();
+        double img1width = image1.getImage().getWidth();
+        double img2height = image2.getImage().getHeight();
+
+        BufferedImage mergedImage = new BufferedImage((int) img1width, (int) (img1height + img2height), BufferedImage.TYPE_INT_RGB);
+        Graphics mergedImageGraphics = mergedImage.getGraphics();
+        try {
+            mergedImageGraphics.drawImage(image1.getImage(), 0, 0, null);
+            mergedImageGraphics.drawImage(image2.getImage(), 0, (int) (img1height), null);
+
+            // set Image, Position and type for merged Image
+            //set position for merged image with values of image1 and the height of both
+            Rectangle2D pos = new Rectangle2D.Float();
+            pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), width, height1 + height2);
+            PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage());
+            // Graphics need to be disposed
+
+            image1.getImage().flush();
+            image2.getImage().flush();
+
+            mergedImage.flush();
+            mergedImageGraphics.dispose();
+
+            return newPdfImage;
+        } catch (Exception e) {
+            // failed to merge image
+            log.error("Failed to merge image", e);
+            return null;
        }

-        document.setPages(pages);

-        classificationService.classifyDocument(document);
-        sectionsBuilderService.buildSections(document);
-        sectionsBuilderService.addImagesToSections(document);
+    }

-        return document;
+    //make a list of true and false, if the image is a candidate for merging
+    private List<Boolean> getCandidatesList(List<PdfImage> imageList) {
+        List<Boolean> candidatesList = new ArrayList<>();
+        for (int i = 0; i < imageList.size(); i++) {
+            if (i >= 1) {
+                candidatesList.add(isCandidateForMerging(imageList.get(i - 1), imageList.get(i)));
+            }
+        }
+        return candidatesList;
+    }
+
+    // evaluate if two images are candidates for merging, depending on their coordinates, width and height
+    private boolean isCandidateForMerging(PdfImage image1, PdfImage image2) {
+        double x1 = image1.getPosition().getX();
+        double y1 = image1.getPosition().getY();
+        double width1 = image1.getPosition().getWidth();
+        double x2 = image2.getPosition().getX();
+        double y2 = image2.getPosition().getY();
+        double width2 = image2.getPosition().getWidth();
+        double height2 = image2.getPosition().getHeight();
+        //if the x-coordinates and widths of images are equal and the height is equal to difference between y-coordinates,
+        // then it is the same picture and has to be merged -> return true
+        return x1 == x2 && width1 == width2 && Math.ceil(height2) == Math.ceil(y1 - y2) && width2 > (height2 / 6);
    }


@@ -116,4 +319,5 @@ public class PdfSegmentationService {

    }

-}
+
+}
@@ -1,29 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;

-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
-import com.iqser.red.service.redaction.v1.server.classification.model.Header;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
+import com.iqser.red.service.redaction.v1.server.classification.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;

@Service
 public class SectionsBuilderService {
@@ -127,6 +113,14 @@ public class SectionsBuilderService {
            paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
        }

+        // first page is always a paragraph, else we can't process pages 1..N,
+        // where N is the first found page with a paragraph
+        if (paragraphMap.get(1) == null) {
+            Paragraph paragraph = new Paragraph();
+            document.getParagraphs().add(paragraph);
+            paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
+        }
+
        for (Page page : document.getPages()) {
            for (PdfImage image : page.getImages()) {
                SortedSet<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
@@ -302,4 +296,4 @@ public class SectionsBuilderService {

    }

-}
+}
@@ -1,17 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.settings;

-import org.springframework.boot.context.properties.ConfigurationProperties;
-
 import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;

@Data
@ConfigurationProperties("redaction-service")
 public class RedactionServiceSettings {
-    
+
    private int numberOfSurroundingWords = 3;

    private int surroundingWordsOffsetWindow = 100;

    private boolean enableImageClassification = true;

-}
+    private float maxImageCropboxRatio = 0.9f;
+
+}
@@ -0,0 +1,103 @@
+package com.iqser.red.service.redaction.v1.server.storage;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.file.management.v1.api.model.FileType;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.server.classification.model.Text;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class RedactionStorageService {
+
+    private final ObjectMapper objectMapper;
+    private final StorageService storageService;
+
+    @SneakyThrows
+    public InputStream getStoredObject(String storageId) {
+        return storageService.getObject(storageId).getInputStream();
+    }
+
+
+    @SneakyThrows
+    public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
+        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any));
+    }
+
+
+    public RedactionLog getRedactionLog(String dossierId, String fileId) {
+
+        InputStreamResource inputStreamResource;
+        try {
+            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Text not available.");
+            return null;
+        }
+
+        try {
+            return objectMapper.readValue(inputStreamResource.getInputStream(), RedactionLog.class);
+        } catch (IOException e) {
+            throw new RuntimeException("Could not convert RedactionLog", e);
+        }
+    }
+
+
+    public Text getText(String dossierId, String fileId) {
+
+        InputStreamResource inputStreamResource;
+        try {
+            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Text not available.");
+            return null;
+        }
+
+        try {
+            return objectMapper.readValue(inputStreamResource.getInputStream(), Text.class);
+        } catch (IOException e) {
+            throw new RuntimeException("Could not convert Text", e);
+        }
+    }
+
+
+    public SectionGrid getSectionGrid(String dossierId, String fileId) {
+
+        var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
+        try {
+            return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class);
+        } catch (IOException e) {
+            throw new RuntimeException("Could not convert RedactionLog", e);
+        }
+    }
+
+
+    @RequiredArgsConstructor
+    public enum StorageType {
+        PARSED_DOCUMENT(".json");
+
+        @Getter
+        private final String extension;
+
+    }
+
+    public static class StorageIdUtils {
+
+        public static String getStorageId(String dossierId, String fileId, FileType fileType) {
+            return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
+        }
+
+    }
+}
@@ -1,6 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;

 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -18,6 +20,8 @@ public abstract class AbstractTextContainer {
    protected String classification;
    protected int page;

+    private Orientation orientation = Orientation.NONE;
+
    public abstract String getText();

    public boolean contains(AbstractTextContainer other) {
@@ -25,15 +29,17 @@ public abstract class AbstractTextContainer {
    }

    public boolean contains(Rectangle other) {
-        return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY()  && this.maxY >= other.getTopLeft().getY() + other.getHeight();
+        return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
    }

+    @JsonIgnore
    public float getHeight() {
        return maxY - minY;
    }
-
+    
+    @JsonIgnore
    public float getWidth() {
        return maxX - minX;
    }

-}
+}
@@ -1,18 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
 import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;

-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
@SuppressWarnings("serial")
@Data
@EqualsAndHashCode(callSuper = true)
@@ -71,7 +70,4 @@ public class Cell extends Rectangle {
    }


-
-
-
-}
+}
@@ -1,10 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

-import java.util.List;
-
 import lombok.Builder;
 import lombok.Data;

+import java.util.List;
+
@Data
@Builder
 public class CleanRulings {
@@ -8,170 +8,171 @@ import java.util.List;
@SuppressWarnings("all")
 public class Rectangle extends Rectangle2D.Float {

-	/**
-	 * Ill-defined comparator, from when Rectangle was Comparable.
-	 * 
-	 * see https://github.com/tabulapdf/tabula-java/issues/116
-	 * @deprecated with no replacement
-	 */
-	@Deprecated
-	public static final Comparator<Rectangle> ILL_DEFINED_ORDER = new Comparator<Rectangle>() {
-		@Override public int compare(Rectangle o1, Rectangle o2) {
-			if (o1.equals(o2)) return 0;
-			if (o1.verticalOverlap(o2) > VERTICAL_COMPARISON_THRESHOLD) {
-				return o1.isLtrDominant() == -1 && o2.isLtrDominant() == -1
-				     ? - java.lang.Double.compare(o1.getX(), o2.getX())
-				     : java.lang.Double.compare(o1.getX(), o2.getX());
-			} else {
-				return java.lang.Float.compare(o1.getBottom(), o2.getBottom());
-			}
-		}
-	};
-	
-	protected static final float VERTICAL_COMPARISON_THRESHOLD = 0.4f;
+    protected static final float VERTICAL_COMPARISON_THRESHOLD = 0.4f;
+    /**
+     * Ill-defined comparator, from when Rectangle was Comparable.
+     * <p>
+     * see https://github.com/tabulapdf/tabula-java/issues/116
+     *
+     * @deprecated with no replacement
+     */
+    @Deprecated
+    public static final Comparator<Rectangle> ILL_DEFINED_ORDER = new Comparator<Rectangle>() {
+        @Override
+        public int compare(Rectangle o1, Rectangle o2) {
+            if (o1.equals(o2)) return 0;
+            if (o1.verticalOverlap(o2) > VERTICAL_COMPARISON_THRESHOLD) {
+                return o1.isLtrDominant() == -1 && o2.isLtrDominant() == -1
+                        ? -java.lang.Double.compare(o1.getX(), o2.getX())
+                        : java.lang.Double.compare(o1.getX(), o2.getX());
+            } else {
+                return java.lang.Float.compare(o1.getBottom(), o2.getBottom());
+            }
+        }
+    };

-	public Rectangle() {
-		super();
-	}
+    public Rectangle() {
+        super();
+    }

-	public Rectangle(float top, float left, float width, float height) {
-		super();
-		this.setRect(left, top, width, height);
-	}
+    public Rectangle(float top, float left, float width, float height) {
+        super();
+        this.setRect(left, top, width, height);
+    }

-	public int compareTo(Rectangle other) {
-		return ILL_DEFINED_ORDER.compare(this, other);
-	}
+    /**
+     * @param rectangles
+     * @return minimum bounding box that contains all the rectangles
+     */
+    public static Rectangle boundingBoxOf(List<? extends Rectangle> rectangles) {
+        float minx = java.lang.Float.MAX_VALUE;
+        float miny = java.lang.Float.MAX_VALUE;
+        float maxx = java.lang.Float.MIN_VALUE;
+        float maxy = java.lang.Float.MIN_VALUE;

-	// I'm bad at Java and need this for fancy sorting in
-	// technology.tabula.TextChunk.
-	public int isLtrDominant() {
-		return 0;
-	}
+        for (Rectangle r : rectangles) {
+            minx = (float) Math.min(r.getMinX(), minx);
+            miny = (float) Math.min(r.getMinY(), miny);
+            maxx = (float) Math.max(r.getMaxX(), maxx);
+            maxy = (float) Math.max(r.getMaxY(), maxy);
+        }
+        return new Rectangle(miny, minx, maxx - minx, maxy - miny);
+    }

-	public float getArea() {
-		return this.width * this.height;
-	}
+    public int compareTo(Rectangle other) {
+        return ILL_DEFINED_ORDER.compare(this, other);
+    }

-	public float verticalOverlap(Rectangle other) {
-		return Math.max(0, Math.min(this.getBottom(), other.getBottom()) - Math.max(this.getTop(), other.getTop()));
-	}
+    // I'm bad at Java and need this for fancy sorting in
+    // technology.tabula.TextChunk.
+    public int isLtrDominant() {
+        return 0;
+    }

-	public boolean verticallyOverlaps(Rectangle other) {
-		return verticalOverlap(other) > 0;
-	}
+    public float getArea() {
+        return this.width * this.height;
+    }

-	public float horizontalOverlap(Rectangle other) {
-		return Math.max(0, Math.min(this.getRight(), other.getRight()) - Math.max(this.getLeft(), other.getLeft()));
-	}
+    public float verticalOverlap(Rectangle other) {
+        return Math.max(0, Math.min(this.getBottom(), other.getBottom()) - Math.max(this.getTop(), other.getTop()));
+    }

-	public boolean horizontallyOverlaps(Rectangle other) {
-		return horizontalOverlap(other) > 0;
-	}
+    public boolean verticallyOverlaps(Rectangle other) {
+        return verticalOverlap(other) > 0;
+    }

-	public float verticalOverlapRatio(Rectangle other) {
-		float rv = 0, delta = Math.min(this.getBottom() - this.getTop(), other.getBottom() - other.getTop());
+    public float horizontalOverlap(Rectangle other) {
+        return Math.max(0, Math.min(this.getRight(), other.getRight()) - Math.max(this.getLeft(), other.getLeft()));
+    }

-		if (other.getTop() <= this.getTop() && this.getTop() <= other.getBottom()
-				&& other.getBottom() <= this.getBottom()) {
-			rv = (other.getBottom() - this.getTop()) / delta;
-		} else if (this.getTop() <= other.getTop() && other.getTop() <= this.getBottom()
-				&& this.getBottom() <= other.getBottom()) {
-			rv = (this.getBottom() - other.getTop()) / delta;
-		} else if (this.getTop() <= other.getTop() && other.getTop() <= other.getBottom()
-				&& other.getBottom() <= this.getBottom()) {
-			rv = (other.getBottom() - other.getTop()) / delta;
-		} else if (other.getTop() <= this.getTop() && this.getTop() <= this.getBottom()
-				&& this.getBottom() <= other.getBottom()) {
-			rv = (this.getBottom() - this.getTop()) / delta;
-		}
+    public boolean horizontallyOverlaps(Rectangle other) {
+        return horizontalOverlap(other) > 0;
+    }

-		return rv;
+    public float verticalOverlapRatio(Rectangle other) {
+        float rv = 0, delta = Math.min(this.getBottom() - this.getTop(), other.getBottom() - other.getTop());

-	}
+        if (other.getTop() <= this.getTop() && this.getTop() <= other.getBottom()
+                && other.getBottom() <= this.getBottom()) {
+            rv = (other.getBottom() - this.getTop()) / delta;
+        } else if (this.getTop() <= other.getTop() && other.getTop() <= this.getBottom()
+                && this.getBottom() <= other.getBottom()) {
+            rv = (this.getBottom() - other.getTop()) / delta;
+        } else if (this.getTop() <= other.getTop() && other.getTop() <= other.getBottom()
+                && other.getBottom() <= this.getBottom()) {
+            rv = (other.getBottom() - other.getTop()) / delta;
+        } else if (other.getTop() <= this.getTop() && this.getTop() <= this.getBottom()
+                && this.getBottom() <= other.getBottom()) {
+            rv = (this.getBottom() - this.getTop()) / delta;
+        }

-	public float overlapRatio(Rectangle other) {
-		double intersectionWidth = Math.max(0,
-				Math.min(this.getRight(), other.getRight()) - Math.max(this.getLeft(), other.getLeft()));
-		double intersectionHeight = Math.max(0,
-				Math.min(this.getBottom(), other.getBottom()) - Math.max(this.getTop(), other.getTop()));
-		double intersectionArea = Math.max(0, intersectionWidth * intersectionHeight);
-		double unionArea = this.getArea() + other.getArea() - intersectionArea;
+        return rv;

-		return (float) (intersectionArea / unionArea);
-	}
+    }

-	public Rectangle merge(Rectangle other) {
-		this.setRect(this.createUnion(other));
-		return this;
-	}
+    public float overlapRatio(Rectangle other) {
+        double intersectionWidth = Math.max(0,
+                Math.min(this.getRight(), other.getRight()) - Math.max(this.getLeft(), other.getLeft()));
+        double intersectionHeight = Math.max(0,
+                Math.min(this.getBottom(), other.getBottom()) - Math.max(this.getTop(), other.getTop()));
+        double intersectionArea = Math.max(0, intersectionWidth * intersectionHeight);
+        double unionArea = this.getArea() + other.getArea() - intersectionArea;

-	public float getTop() {
-		return (float) this.getMinY();
-	}
+        return (float) (intersectionArea / unionArea);
+    }

-	public void setTop(float top) {
-		float deltaHeight = top - this.y;
-		this.setRect(this.x, top, this.width, this.height - deltaHeight);
-	}
+    public Rectangle merge(Rectangle other) {
+        this.setRect(this.createUnion(other));
+        return this;
+    }

-	public float getRight() {
-		return (float) this.getMaxX();
-	}
+    public float getTop() {
+        return (float) this.getMinY();
+    }

-	public void setRight(float right) {
-		this.setRect(this.x, this.y, right - this.x, this.height);
-	}
+    public void setTop(float top) {
+        float deltaHeight = top - this.y;
+        this.setRect(this.x, top, this.width, this.height - deltaHeight);
+    }

-	public float getLeft() {
-		return (float) this.getMinX();
-	}
+    public float getRight() {
+        return (float) this.getMaxX();
+    }

-	public void setLeft(float left) {
-		float deltaWidth = left - this.x;
-		this.setRect(left, this.y, this.width - deltaWidth, this.height);
-	}
+    public void setRight(float right) {
+        this.setRect(this.x, this.y, right - this.x, this.height);
+    }

-	public float getBottom() {
-		return (float) this.getMaxY();
-	}
+    public float getLeft() {
+        return (float) this.getMinX();
+    }

-	public void setBottom(float bottom) {
-		this.setRect(this.x, this.y, this.width, bottom - this.y);
-	}
+    public void setLeft(float left) {
+        float deltaWidth = left - this.x;
+        this.setRect(left, this.y, this.width - deltaWidth, this.height);
+    }

-	public Point2D[] getPoints() {
-		return new Point2D[] { new Point2D.Float(this.getLeft(), this.getTop()),
-				new Point2D.Float(this.getRight(), this.getTop()), new Point2D.Float(this.getRight(), this.getBottom()),
-				new Point2D.Float(this.getLeft(), this.getBottom()) };
-	}
+    public float getBottom() {
+        return (float) this.getMaxY();
+    }

-	@Override
-	public String toString() {
-		StringBuilder sb = new StringBuilder();
-		String s = super.toString();
-		sb.append(s.substring(0, s.length() - 1));
-		sb.append(String.format(",bottom=%f,right=%f]", this.getBottom(), this.getRight()));
-		return sb.toString();
-	}
+    public void setBottom(float bottom) {
+        this.setRect(this.x, this.y, this.width, bottom - this.y);
+    }

-	/**
-	 * @param rectangles
-	 * @return minimum bounding box that contains all the rectangles
-	 */
-	public static Rectangle boundingBoxOf(List<? extends Rectangle> rectangles) {
-		float minx = java.lang.Float.MAX_VALUE;
-		float miny = java.lang.Float.MAX_VALUE;
-		float maxx = java.lang.Float.MIN_VALUE;
-		float maxy = java.lang.Float.MIN_VALUE;
+    public Point2D[] getPoints() {
+        return new Point2D[]{new Point2D.Float(this.getLeft(), this.getTop()),
+                new Point2D.Float(this.getRight(), this.getTop()), new Point2D.Float(this.getRight(), this.getBottom()),
+                new Point2D.Float(this.getLeft(), this.getBottom())};
+    }

-		for (Rectangle r : rectangles) {
-			minx = (float) Math.min(r.getMinX(), minx);
-			miny = (float) Math.min(r.getMinY(), miny);
-			maxx = (float) Math.max(r.getMaxX(), maxx);
-			maxy = (float) Math.max(r.getMaxY(), maxy);
-		}
-		return new Rectangle(miny, minx, maxx - minx, maxy - miny);
-	}
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        String s = super.toString();
+        sb.append(s.substring(0, s.length() - 1));
+        sb.append(String.format(",bottom=%f,right=%f]", this.getBottom(), this.getRight()));
+        return sb.toString();
+    }

 }
@@ -1,12 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

-import java.util.ArrayList;
-import java.util.List;
-
+import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
 import org.locationtech.jts.geom.Envelope;
 import org.locationtech.jts.index.strtree.STRtree;

-import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
+import java.util.ArrayList;
+import java.util.List;

@SuppressWarnings("all")
 public class RectangleSpatialIndex<T extends Rectangle> {
@@ -1,20 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

+import com.iqser.red.service.redaction.v1.server.tableextraction.utils.CohenSutherlandClipping;
+import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
+import lombok.extern.slf4j.Slf4j;
+
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Formatter;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import com.iqser.red.service.redaction.v1.server.tableextraction.utils.CohenSutherlandClipping;
-import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
-
-import lombok.extern.slf4j.Slf4j;
+import java.util.*;


@Slf4j
@@ -23,231 +16,10 @@ public class Ruling extends Line2D.Float {

    private static int PERPENDICULAR_PIXEL_EXPAND_AMOUNT = 2;

-    private enum SOType {VERTICAL, HRIGHT, HLEFT}
-
-
    public Ruling(Point2D p1, Point2D p2) {
        super(p1, p2);
    }

-    public boolean vertical() {
-        return this.length() > 0 && Utils.feq(this.x1, this.x2); //diff < ORIENTATION_CHECK_THRESHOLD;
-    }
-
-    public boolean horizontal() {
-        return this.length() > 0 && Utils.feq(this.y1, this.y2); //diff < ORIENTATION_CHECK_THRESHOLD;
-    }
-
-    public boolean oblique() {
-        return !(this.vertical() || this.horizontal());
-    }
-
-    // attributes that make sense only for non-oblique lines
-    // these are used to have a single collapse method (in page, currently)
-
-    public float getPosition() {
-        if (this.oblique()) {
-            throw new UnsupportedOperationException();
-        }
-        return this.vertical() ? this.getLeft() : this.getTop();
-    }
-
-
-    public float getStart() {
-        if (this.oblique()) {
-            throw new UnsupportedOperationException();
-        }
-        return this.vertical() ? this.getTop() : this.getLeft();
-    }
-
-    public void setStart(float v) {
-        if (this.oblique()) {
-            throw new UnsupportedOperationException();
-        }
-        if (this.vertical()) {
-            this.setTop(v);
-        } else {
-            this.setLeft(v);
-        }
-    }
-
-    public float getEnd() {
-        if (this.oblique()) {
-            throw new UnsupportedOperationException();
-        }
-        return this.vertical() ? this.getBottom() : this.getRight();
-    }
-
-    public void setEnd(float v) {
-        if (this.oblique()) {
-            throw new UnsupportedOperationException();
-        }
-        if (this.vertical()) {
-            this.setBottom(v);
-        } else {
-            this.setRight(v);
-        }
-    }
-
-    public void setStartEnd(float start, float end) {
-        if (this.oblique()) {
-            throw new UnsupportedOperationException();
-        }
-        if (this.vertical()) {
-            this.setTop(start);
-            this.setBottom(end);
-        } else {
-            this.setLeft(start);
-            this.setRight(end);
-        }
-    }
-
-
-    public boolean perpendicularTo(Ruling other) {
-        return this.vertical() == other.horizontal();
-    }
-
-
-    public boolean nearlyIntersects(Ruling another, int colinearOrParallelExpandAmount) {
-        if (this.intersectsLine(another)) {
-            return true;
-        }
-
-        boolean rv = false;
-
-        if (this.perpendicularTo(another)) {
-            rv = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT).intersectsLine(another);
-        } else {
-            rv = this.expand(colinearOrParallelExpandAmount)
-                    .intersectsLine(another.expand(colinearOrParallelExpandAmount));
-        }
-
-        return rv;
-    }
-
-    public double length() {
-        return Math.sqrt(Math.pow(this.x1 - this.x2, 2) + Math.pow(this.y1 - this.y2, 2));
-    }
-
-    public Ruling intersect(Rectangle2D clip) {
-        Float clipee = (Float) this.clone();
-        boolean clipped = new CohenSutherlandClipping(clip).clip(clipee);
-
-        if (clipped) {
-            return new Ruling(clipee.getP1(), clipee.getP2());
-        } else {
-            return this;
-        }
-    }
-
-    public Ruling expand(float amount) {
-        Ruling r = (Ruling) this.clone();
-        r.setStart(this.getStart() - amount);
-        r.setEnd(this.getEnd() + amount);
-        return r;
-    }
-
-    public Point2D intersectionPoint(Ruling other) {
-        Ruling this_l = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
-        Ruling other_l = other.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
-        Ruling horizontal, vertical;
-
-        if (!this_l.intersectsLine(other_l)) {
-            return null;
-        }
-
-        if (this_l.horizontal() && other_l.vertical()) {
-            horizontal = this_l;
-            vertical = other_l;
-        } else if (this_l.vertical() && other_l.horizontal()) {
-            vertical = this_l;
-            horizontal = other_l;
-        } else {
-            log.warn("lines must be orthogonal, vertical and horizontal");
-            return null;
-        }
-        return new Point2D.Float(vertical.getLeft(), horizontal.getTop());
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        if (this == other) {
-            return true;
-        }
-
-        if (!(other instanceof Ruling)) {
-            return false;
-        }
-
-        Ruling o = (Ruling) other;
-        return this.getP1().equals(o.getP1()) && this.getP2().equals(o.getP2());
-    }
-
-    @Override
-    public int hashCode() {
-        return super.hashCode();
-    }
-
-    public float getTop() {
-        return this.y1;
-    }
-
-    public void setTop(float v) {
-        setLine(this.getLeft(), v, this.getRight(), this.getBottom());
-    }
-
-    public float getLeft() {
-        return this.x1;
-    }
-
-    public void setLeft(float v) {
-        setLine(v, this.getTop(), this.getRight(), this.getBottom());
-    }
-
-    public float getBottom() {
-        return this.y2;
-    }
-
-    public void setBottom(float v) {
-        setLine(this.getLeft(), this.getTop(), this.getRight(), v);
-    }
-
-    public float getRight() {
-        return this.x2;
-    }
-
-    public void setRight(float v) {
-        setLine(this.getLeft(), this.getTop(), v, this.getBottom());
-    }
-
-    public float getWidth() {
-        return this.getRight() - this.getLeft();
-    }
-
-    public float getHeight() {
-        return this.getBottom() - this.getTop();
-    }
-
-    public double getAngle() {
-        double angle = Math.toDegrees(Math.atan2(this.getP2().getY() - this.getP1().getY(),
-                this.getP2().getX() - this.getP1().getX()));
-
-        if (angle < 0) {
-            angle += 360;
-        }
-        return angle;
-    }
-
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        Formatter formatter = new Formatter(sb);
-        String rv = formatter.format("%s[minX=%f minY=%f maxX=%f maxY=%f]", this.getClass().toString(), this.x1, this.y1, this.x2, this.y2).toString();
-        formatter.close();
-        return rv;
-    }
-
    public static List<Ruling> cropRulingsToArea(List<Ruling> rulings, Rectangle2D area) {
        ArrayList<Ruling> rv = new ArrayList<>();
        for (Ruling r : rulings) {
@@ -346,7 +118,7 @@ public class Ruling extends Line2D.Float {
                            rv.put(i,
                                    new Ruling[]{h.getKey().expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT),
                                            so.ruling.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)});
-                        } catch(UnsupportedOperationException e){
+                        } catch (UnsupportedOperationException e) {
                            log.info("Some line are oblique, ignoring...");
                            continue;
                        }
@@ -365,5 +137,225 @@ public class Ruling extends Line2D.Float {

    }

+    public boolean vertical() {
+        return this.length() > 0 && Utils.feq(this.x1, this.x2); //diff < ORIENTATION_CHECK_THRESHOLD;
+    }
+
+    public boolean horizontal() {
+        return this.length() > 0 && Utils.feq(this.y1, this.y2); //diff < ORIENTATION_CHECK_THRESHOLD;
+    }
+
+    // attributes that make sense only for non-oblique lines
+    // these are used to have a single collapse method (in page, currently)
+
+    public boolean oblique() {
+        return !(this.vertical() || this.horizontal());
+    }
+
+    public float getPosition() {
+        if (this.oblique()) {
+            throw new UnsupportedOperationException();
+        }
+        return this.vertical() ? this.getLeft() : this.getTop();
+    }
+
+    public float getStart() {
+        if (this.oblique()) {
+            throw new UnsupportedOperationException();
+        }
+        return this.vertical() ? this.getTop() : this.getLeft();
+    }
+
+    public void setStart(float v) {
+        if (this.oblique()) {
+            throw new UnsupportedOperationException();
+        }
+        if (this.vertical()) {
+            this.setTop(v);
+        } else {
+            this.setLeft(v);
+        }
+    }
+
+    public float getEnd() {
+        if (this.oblique()) {
+            throw new UnsupportedOperationException();
+        }
+        return this.vertical() ? this.getBottom() : this.getRight();
+    }
+
+    public void setEnd(float v) {
+        if (this.oblique()) {
+            throw new UnsupportedOperationException();
+        }
+        if (this.vertical()) {
+            this.setBottom(v);
+        } else {
+            this.setRight(v);
+        }
+    }
+
+    public void setStartEnd(float start, float end) {
+        if (this.oblique()) {
+            throw new UnsupportedOperationException();
+        }
+        if (this.vertical()) {
+            this.setTop(start);
+            this.setBottom(end);
+        } else {
+            this.setLeft(start);
+            this.setRight(end);
+        }
+    }
+
+    public boolean perpendicularTo(Ruling other) {
+        return this.vertical() == other.horizontal();
+    }
+
+    public boolean nearlyIntersects(Ruling another, int colinearOrParallelExpandAmount) {
+        if (this.intersectsLine(another)) {
+            return true;
+        }
+
+        boolean rv = false;
+
+        if (this.perpendicularTo(another)) {
+            rv = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT).intersectsLine(another);
+        } else {
+            rv = this.expand(colinearOrParallelExpandAmount)
+                    .intersectsLine(another.expand(colinearOrParallelExpandAmount));
+        }
+
+        return rv;
+    }
+
+    public double length() {
+        return Math.sqrt(Math.pow(this.x1 - this.x2, 2) + Math.pow(this.y1 - this.y2, 2));
+    }
+
+    public Ruling intersect(Rectangle2D clip) {
+        Float clipee = (Float) this.clone();
+        boolean clipped = new CohenSutherlandClipping(clip).clip(clipee);
+
+        if (clipped) {
+            return new Ruling(clipee.getP1(), clipee.getP2());
+        } else {
+            return this;
+        }
+    }
+
+    public Ruling expand(float amount) {
+        Ruling r = (Ruling) this.clone();
+        try {
+            r.setStart(this.getStart() - amount);
+            r.setEnd(this.getEnd() + amount);
+        } catch (UnsupportedOperationException e){
+            log.warn("Could not expand ruling!");
+        }
+        return r;
+    }
+
+    public Point2D intersectionPoint(Ruling other) {
+        Ruling this_l = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
+        Ruling other_l = other.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
+        Ruling horizontal, vertical;
+
+        if (!this_l.intersectsLine(other_l)) {
+            return null;
+        }
+
+        if (this_l.horizontal() && other_l.vertical()) {
+            horizontal = this_l;
+            vertical = other_l;
+        } else if (this_l.vertical() && other_l.horizontal()) {
+            vertical = this_l;
+            horizontal = other_l;
+        } else {
+            log.warn("lines must be orthogonal, vertical and horizontal");
+            return null;
+        }
+        return new Point2D.Float(vertical.getLeft(), horizontal.getTop());
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (!(other instanceof Ruling)) {
+            return false;
+        }
+
+        Ruling o = (Ruling) other;
+        return this.getP1().equals(o.getP1()) && this.getP2().equals(o.getP2());
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+
+    public float getTop() {
+        return this.y1;
+    }
+
+    public void setTop(float v) {
+        setLine(this.getLeft(), v, this.getRight(), this.getBottom());
+    }
+
+    public float getLeft() {
+        return this.x1;
+    }
+
+    public void setLeft(float v) {
+        setLine(v, this.getTop(), this.getRight(), this.getBottom());
+    }
+
+    public float getBottom() {
+        return this.y2;
+    }
+
+    public void setBottom(float v) {
+        setLine(this.getLeft(), this.getTop(), this.getRight(), v);
+    }
+
+    public float getRight() {
+        return this.x2;
+    }
+
+    public void setRight(float v) {
+        setLine(this.getLeft(), this.getTop(), v, this.getBottom());
+    }
+
+    public float getWidth() {
+        return this.getRight() - this.getLeft();
+    }
+
+    public float getHeight() {
+        return this.getBottom() - this.getTop();
+    }
+
+    public double getAngle() {
+        double angle = Math.toDegrees(Math.atan2(this.getP2().getY() - this.getP1().getY(),
+                this.getP2().getX() - this.getP1().getX()));
+
+        if (angle < 0) {
+            angle += 360;
+        }
+        return angle;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        Formatter formatter = new Formatter(sb);
+        String rv = formatter.format("%s[minX=%f minY=%f maxX=%f maxY=%f]", this.getClass().toString(), this.x1, this.y1, this.x2, this.y2).toString();
+        formatter.close();
+        return rv;
+    }
+
+    private enum SOType {VERTICAL, HRIGHT, HLEFT}
+

 }
@@ -1,22 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.commons.collections4.CollectionUtils;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
-
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.util.*;

@Slf4j
 public class Table extends AbstractTextContainer {
@@ -24,21 +15,14 @@ public class Table extends AbstractTextContainer {
    private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();

    private final RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();
-
+    private final int rotation;
    @Getter
    @Setter
    private String headline;
-
    private int unrotatedRowCount;
-
    private int unrotatedColCount;
-
    private int rowCount = -1;
-
    private int colCount = -1;
-
-    private final int rotation;
-
    private List<List<Cell>> rows;


@@ -62,8 +46,8 @@ public class Table extends AbstractTextContainer {

            // Ignore rows that does not contain any cells and values.
            List<List<Cell>> rowsToRemove = new ArrayList<>();
-            for (List<Cell> row: rows){
-                if (row.size() == 1 && row.get(0).getTextBlocks().isEmpty()){
+            for (List<Cell> row : rows) {
+                if (row.size() == 1 && row.get(0).getTextBlocks().isEmpty()) {
                    rowsToRemove.add(row);
                }
            }
@@ -110,7 +94,7 @@ public class Table extends AbstractTextContainer {
        // we move from left to right and top to bottom
        for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
            List<Cell> rowCells = rows.get(rowIndex);
-            if(rowCells.size() == 1){
+            if (rowCells.size() == 1) {
                continue;
            }

@@ -275,7 +259,7 @@ public class Table extends AbstractTextContainer {

        cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2),
                Utils.round(arg1
-                .getBottom(), 2))));
+                        .getBottom(), 2))));

        Iterator<Cell> iter = cells.iterator();
        Cell c = iter.next();
@@ -367,4 +351,4 @@ public class Table extends AbstractTextContainer {
        return sb.toString();
    }

-}
+}
@@ -1,19 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.service;

-import java.awt.geom.Line2D;
-import java.awt.geom.Point2D;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
+import org.springframework.stereotype.Service;
+
+import java.awt.geom.Line2D;
+import java.awt.geom.Point2D;
+import java.util.*;

@Service
 public class RulingCleaningService {
@@ -1,31 +1,57 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.service;

-import java.awt.geom.Point2D;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.*;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
+import org.springframework.stereotype.Service;
+
+import java.awt.geom.Point2D;
+import java.util.*;
+import java.util.stream.Collectors;

@Service
 public class TableExtractionService {

+    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (arg0, arg1) -> {
+
+        int rv = 0;
+        float arg0X = Utils.round(arg0.getX(), 2);
+        float arg0Y = Utils.round(arg0.getY(), 2);
+        float arg1X = Utils.round(arg1.getX(), 2);
+        float arg1Y = Utils.round(arg1.getY(), 2);
+
+        if (arg0X > arg1X) {
+            rv = 1;
+        } else if (arg0X < arg1X) {
+            rv = -1;
+        } else if (arg0Y > arg1Y) {
+            rv = 1;
+        } else if (arg0Y < arg1Y) {
+            rv = -1;
+        }
+        return rv;
+    };
+    private static final Comparator<Point2D> POINT_COMPARATOR = (arg0, arg1) -> {
+
+        int rv = 0;
+        float arg0X = Utils.round(arg0.getX(), 2);
+        float arg0Y = Utils.round(arg0.getY(), 2);
+        float arg1X = Utils.round(arg1.getX(), 2);
+        float arg1Y = Utils.round(arg1.getY(), 2);
+
+        if (arg0Y > arg1Y) {
+            rv = 1;
+        } else if (arg0Y < arg1Y) {
+            rv = -1;
+        } else if (arg0X > arg1X) {
+            rv = 1;
+        } else if (arg0X < arg1X) {
+            rv = -1;
+        }
+        return rv;
+    };
+
    public void extractTables(CleanRulings cleanRulings, Page page) {

        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
@@ -80,7 +106,6 @@ public class TableExtractionService {
        page.getTextBlocks().removeAll(toBeRemoved);
    }

-
    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

        List<Cell> cellsFound = new ArrayList<>();
@@ -133,7 +158,6 @@ public class TableExtractionService {
        return cellsFound;
    }

-
    private List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
        // via: http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon
        List<Rectangle> rectangles = new ArrayList<>();
@@ -233,47 +257,6 @@ public class TableExtractionService {
        return rectangles;
    }

-
-    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (arg0, arg1) -> {
-
-        int rv = 0;
-        float arg0X = Utils.round(arg0.getX(), 2);
-        float arg0Y = Utils.round(arg0.getY(), 2);
-        float arg1X = Utils.round(arg1.getX(), 2);
-        float arg1Y = Utils.round(arg1.getY(), 2);
-
-        if (arg0X > arg1X) {
-            rv = 1;
-        } else if (arg0X < arg1X) {
-            rv = -1;
-        } else if (arg0Y > arg1Y) {
-            rv = 1;
-        } else if (arg0Y < arg1Y) {
-            rv = -1;
-        }
-        return rv;
-    };
-
-    private static final Comparator<Point2D> POINT_COMPARATOR = (arg0, arg1) -> {
-
-        int rv = 0;
-        float arg0X = Utils.round(arg0.getX(), 2);
-        float arg0Y = Utils.round(arg0.getY(), 2);
-        float arg1X = Utils.round(arg1.getX(), 2);
-        float arg1Y = Utils.round(arg1.getY(), 2);
-
-        if (arg0Y > arg1Y) {
-            rv = 1;
-        } else if (arg0Y < arg1Y) {
-            rv = -1;
-        } else if (arg0X > arg1X) {
-            rv = 1;
-        } else if (arg0X < arg1X) {
-            rv = -1;
-        }
-        return rv;
-    };
-
    private enum Direction {
        HORIZONTAL, VERTICAL
    }
@@ -19,21 +19,24 @@ import java.awt.geom.Rectangle2D;
 * clipping algorithm (line against clip rectangle).
 */
@SuppressWarnings("all")
-public final class CohenSutherlandClipping
-{
+public final class CohenSutherlandClipping {
+    private static final int INSIDE = 0;
+    private static final int LEFT = 1;
+    private static final int RIGHT = 2;
+    private static final int BOTTOM = 4;
+    private static final int TOP = 8;
    private double xMin;
    private double yMin;
    private double xMax;
    private double yMax;
-
    /**
     * Creates a Cohen Sutherland clipper with clip rect (0, 0, 0, 0).
     */
    public CohenSutherlandClipping() {
    }
-
    /**
     * Creates a Cohen Sutherland clipper with the given clip rectangle.
+     *
     * @param clip the clip rectangle to use
     */
    public CohenSutherlandClipping(Rectangle2D clip) {
@@ -42,6 +45,7 @@ public final class CohenSutherlandClipping

    /**
     * Sets the clip rectangle.
+     *
     * @param clip the clip rectangle
     */
    public void setClip(Rectangle2D clip) {
@@ -51,19 +55,13 @@ public final class CohenSutherlandClipping
        yMax = yMin + clip.getHeight();
    }

-    private static final int INSIDE = 0;
-    private static final int LEFT   = 1;
-    private static final int RIGHT  = 2;
-    private static final int BOTTOM = 4;
-    private static final int TOP    = 8;
-
    private final int regionCode(double x, double y) {
-        int code = x < xMin 
-            ? LEFT
-            : x > xMax
+        int code = x < xMin
+                ? LEFT
+                : x > xMax
                ? RIGHT
                : INSIDE;
-             if (y < yMin) code |= BOTTOM;
+        if (y < yMin) code |= BOTTOM;
        else if (y > yMax) code |= TOP;
        return code;
    }
@@ -71,6 +69,7 @@ public final class CohenSutherlandClipping
    /**
     * Clips a given line against the clip rectangle.
     * The modification (if needed) is done in place.
+     *
     * @param line the line to clip
     * @return true if line is clipped, false if line is
     * totally outside the clip rect.
@@ -87,9 +86,9 @@ public final class CohenSutherlandClipping

        boolean vertical = p1x == p2x;

-        double slope = vertical 
-            ? 0d
-            : (p2y-p1y)/(p2x-p1x);
+        double slope = vertical
+                ? 0d
+                : (p2y - p1y) / (p2x - p1x);

        int c1 = regionCode(p1x, p1y);
        int c2 = regionCode(p2x, p2y);
@@ -103,31 +102,27 @@ public final class CohenSutherlandClipping

            if ((c & LEFT) != INSIDE) {
                qx = xMin;
-                qy = (Utils.feq(qx, p1x) ? 0 : qx-p1x)*slope + p1y;
-            }
-            else if ((c & RIGHT) != INSIDE) {
+                qy = (Utils.feq(qx, p1x) ? 0 : qx - p1x) * slope + p1y;
+            } else if ((c & RIGHT) != INSIDE) {
                qx = xMax;
-                qy = (Utils.feq(qx, p1x) ? 0 : qx-p1x)*slope + p1y;
-            }
-            else if ((c & BOTTOM) != INSIDE) {
+                qy = (Utils.feq(qx, p1x) ? 0 : qx - p1x) * slope + p1y;
+            } else if ((c & BOTTOM) != INSIDE) {
                qy = yMin;
                qx = vertical
-                    ? p1x
-                    : (Utils.feq(qy, p1y) ? 0 : qy-p1y)/slope + p1x;
-            }
-            else if ((c & TOP) != INSIDE) {
+                        ? p1x
+                        : (Utils.feq(qy, p1y) ? 0 : qy - p1y) / slope + p1x;
+            } else if ((c & TOP) != INSIDE) {
                qy = yMax;
                qx = vertical
-                    ? p1x
-                    : (Utils.feq(qy, p1y) ? 0 : qy-p1y)/slope + p1x;
+                        ? p1x
+                        : (Utils.feq(qy, p1y) ? 0 : qy - p1y) / slope + p1x;
            }

            if (c == c1) {
                p1x = qx;
                p1y = qy;
-                c1  = regionCode(p1x, p1y);
-            }
-            else {
+                c1 = regionCode(p1x, p1y);
+            } else {
                p2x = qx;
                p2y = qy;
                c2 = regionCode(p2x, p2y);
@@ -137,4 +132,4 @@ public final class CohenSutherlandClipping
        return true;
    }
 }
-// end of file
+// end of file
@@ -10,11 +10,6 @@ import java.util.List;
 */
 public final class QuickSort {

-    private QuickSort() {
-
-    }
-
-
    private static final Comparator<? extends Comparable> OBJCOMP = new Comparator<Comparable>() {
        @Override
        public int compare(Comparable object1, Comparable object2) {
@@ -24,6 +19,10 @@ public final class QuickSort {
    };


+    private QuickSort() {
+
+    }
+
    /**
     * Sorts the given list using the given comparator.
     *
@@ -1,11 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.utils;

+import lombok.extern.slf4j.Slf4j;
+
 import java.math.BigDecimal;
 import java.util.Comparator;
 import java.util.List;

-import lombok.extern.slf4j.Slf4j;
-
@Slf4j
@SuppressWarnings("all")
 public class Utils {
@@ -1,15 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.visualization.service;

-import java.awt.Color;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.PDPageContentStream;
-import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
@@ -17,9 +7,17 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.springframework.stereotype.Service;
+
+import java.awt.Color;
+import java.io.IOException;
+import java.util.List;

@Slf4j
@Service
@@ -34,7 +32,7 @@ public class PdfVisualisationService {
            PDPage pdPage = document.getPage(page - 1);
            PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);

-            for(Paragraph paragraph : classifiedDoc.getParagraphs()) {
+            for (Paragraph paragraph : classifiedDoc.getParagraphs()) {

                for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {

@@ -44,10 +42,10 @@ public class PdfVisualisationService {
                        continue;
                    }
                    if (textBlock instanceof TextBlock) {
-                        textBlock.setClassification((i+1) + "/" + paragraph.getPageBlocks().size());
+                        textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
                        visualizeTextBlock((TextBlock) textBlock, contentStream);
                    } else if (textBlock instanceof Table) {
-                        textBlock.setClassification((i+1) + "/" + paragraph.getPageBlocks().size());
+                        textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
                        visualizeTable((Table) textBlock, contentStream);
                    }

@@ -59,7 +57,6 @@ public class PdfVisualisationService {
    }


-
    public void visualizeClassifications(Document classifiedDoc, PDDocument document) throws IOException {

        for (int page = 1; page <= document.getNumberOfPages(); page++) {
@@ -105,7 +102,7 @@ public class PdfVisualisationService {

            contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());

-            contentStream.showText(textBlock.getClassification());
+            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation());

            contentStream.endText();
        }
@@ -1,4 +1,11 @@
 server:
  port: 8083

-configuration-service.url: "http://localhost:8081"
+configuration-service.url: "http://localhost:8081"
+file-management-service.url: "http://localhost:8085"
+
+storage:
+  bucket-name: 'redaction'
+  endpoint: 'http://localhost:9000'
+  key: minioadmin
+  secret: minioadmin
@@ -2,6 +2,7 @@ info:
  description: Redaction Service Server V1

 configuration-service.url: "http://configuration-service-v1:8080"
+file-management-service.url: "http://file-management-service-v1:8080"
 image-service.url: "http://image-service-v1:8080"

 server:
@@ -10,6 +11,20 @@ server:
 spring:
  profiles:
    active: kubernetes
+  rabbitmq:
+    host: ${RABBITMQ_HOST:localhost}
+    port: ${RABBITMQ_PORT:5672}
+    username: ${RABBITMQ_USERNAME:user}
+    password: ${RABBITMQ_PASSWORD:rabbitmq}
+    listener:
+      simple:
+        acknowledge-mode: AUTO
+        concurrency: 2
+        retry:
+          enabled: true
+          max-attempts: 3
+          max-interval: 15000
+        prefetch: 1

 management:
  endpoint:
@@ -17,4 +32,11 @@ management:
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
  endpoints.web.exposure.include: prometheus, health
-  metrics.export.prometheus.enabled: ${monitoring.enabled:false}
+  metrics.export.prometheus.enabled: ${monitoring.enabled:false}
+
+
+storage:
+  signer-type: 'AWSS3V4SignerType'
+  bucket-name: 'redaction'
+  region: 'us-east-1'
+  endpoint: 'https://s3.amazonaws.com'
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Dominique Eifländer	6bd8c2cf0f	Fixed endless processing	2021-06-02 13:47:48 +02:00
Dominique Eifländer	418faac923	Ignore failing Ruling expansion	2021-06-02 12:53:30 +02:00
Dominique Eiflaender	2ef4e7889e	Pull request #165 : Fixed text parsing orientation problem Merge in RED/redaction-service from ParsingOrientationFix to master * commit 'ab8a710318802d4a836c006638dddb987f5d29c2': Fixed text parsing orientation problem	2021-06-02 10:58:00 +02:00
Dominique Eifländer	ab8a710318	Fixed text parsing orientation problem	2021-06-02 10:47:40 +02:00
Dominique Eiflaender	1556dbad10	Pull request #164 : RED-1535: Improved document parsing, added orientation to textblock Merge in RED/redaction-service from RED-1535 to master * commit '53ce6cb47cd5806e1880dcfeaec173e2efed13ff': RED-1535: Improved document parsing, added orientation to textblock	2021-06-01 13:47:59 +02:00
Dominique Eifländer	53ce6cb47c	RED-1535: Improved document parsing, added orientation to textblock	2021-06-01 13:41:22 +02:00
Timo Bejan	837f97c157	Pull request #163 : rename project and ruleSet - finalization with dependencies Merge in RED/redaction-service from RED-1329 to master * commit 'f60b0f25e579da2270e6925cca78b51ccdc71b38': rename project and ruleSet - finalization with dependencies	2021-05-31 14:21:57 +02:00
Timo	f60b0f25e5	rename project and ruleSet - finalization with dependencies	2021-05-31 15:04:15 +03:00
Timo Bejan	3995c9a3e7	Pull request #162 : migrated ruleset and projectid to new naming Merge in RED/redaction-service from RED-1329 to master * commit 'ceaaed4fc721c9488582d5fce223a9ced928862a': migrated ruleset and projectid to new naming migrated ruleset and projectid to new naming	2021-05-31 10:44:17 +02:00
Timo	ceaaed4fc7	migrated ruleset and projectid to new naming	2021-05-31 11:40:02 +03:00
Timo	e01074ef27	migrated ruleset and projectid to new naming	2021-05-31 10:56:48 +03:00
Dominique Eiflaender	b7d399c461	Pull request #161 : Fixed missing whitespaces Merge in RED/redaction-service from fixMissingSpaces to master * commit '35dec94ccdfc569f24d6983c5b2810dfd0f80e7c': Fixed missing whitespaces	2021-05-28 15:04:47 +02:00
Dominique Eifländer	35dec94ccd	Fixed missing whitespaces	2021-05-28 14:43:30 +02:00
Dominique Eiflaender	42f08194fd	Pull request #160 : RED-1536: Fixed changelog creation Merge in RED/redaction-service from RED-1536 to master * commit 'dc1dba00c11f4778dc5483f47fc147b258b830d2': RED-1536: Fixed changelog creation	2021-05-25 13:52:24 +02:00
Dominique Eifländer	dc1dba00c1	RED-1536: Fixed changelog creation	2021-05-25 13:45:33 +02:00
Christoph Schabert	a900f5b617	Pull request #159 : Swich to maven buld script Merge in RED/redaction-service from RED-1533 to master * commit 'd614fa6001dd46b82da63e219fe924d7152768e7': Swich to maven buld script	2021-05-25 08:49:15 +02:00
cschabert	d614fa6001	Swich to maven buld script	2021-05-21 16:36:17 +02:00
Dominique Eiflaender	e9dcd3535f	Pull request #158 : Ignore images that are contained in others Merge in RED/redaction-service from ignoreInnerImages to master * commit 'd87cbdeaeb87f9767fed876076fdec223824f3f6': Ignore images that are contained in others	2021-05-20 10:33:45 +02:00
Dominique Eifländer	d87cbdeaeb	Ignore images that are contained in others	2021-05-20 09:20:22 +02:00
Dominique Eiflaender	8ece616229	Pull request #157 : RED-1390: Do not classify images that are nearly as big as the page Merge in RED/redaction-service from RED-1390 to master * commit '13974d13734cc9c43e44338bef885a1ace2e5f50': RED-1390: Do not classify images that are nearly as big as the page	2021-05-19 12:03:43 +02:00
Dominique Eifländer	13974d1373	RED-1390: Do not classify images that are nearly as big as the page	2021-05-19 11:46:08 +02:00
Dominique Eiflaender	63229316c8	Pull request #156 : RED-1506: Fixed unpocess images in strange formats Merge in RED/redaction-service from RED-1506 to master * commit '816f44edf0e4a5fe9685c978850b08c09fd083ca': RED-1506: Fixed unpocess images in strange formats	2021-05-19 10:21:59 +02:00
Dominique Eifländer	816f44edf0	RED-1506: Fixed unpocess images in strange formats	2021-05-19 10:18:11 +02:00
Timo Bejan	be92e3c036	Pull request #155 : RED-785 Merge in RED/redaction-service from RED-785 to master * commit '513501abf476dc7f08446bb3035da888d2fd30a2': save legal basis version on redaction log adjusted tests legal basis is now stored in redaction log	2021-05-14 14:07:56 +02:00
Timo	2334a6cb5b	save legal basis version on redaction log	2021-05-14 15:07:07 +03:00
Timo	513501abf4	save legal basis version on redaction log	2021-05-14 15:00:22 +03:00
Timo	6a660c6a73	adjusted tests	2021-05-14 14:50:06 +03:00
Timo	8b898a9038	legal basis is now stored in redaction log	2021-05-14 14:49:21 +03:00
Dominique Eiflaender	b16b56daf1	Pull request #154 : RED-1477: Fixed duplicate images Merge in RED/redaction-service from RED-1477 to master * commit 'bdf40f5e4eec9984306516790eb8c4eb106b0aa1': RED-1477: Fixed duplicate images	2021-05-12 12:23:03 +02:00
Dominique Eifländer	bdf40f5e4e	RED-1477: Fixed duplicate images	2021-05-12 11:28:17 +02:00
Dominique Eiflaender	e281ab98fa	Pull request #153 : Fixed dossier dictionary renalaysis Merge in RED/redaction-service from dossierReanalysisFix to master * commit '184f55535aa07123a80a78e7a0aff949c2827465': Fixed dossier dictionary renalaysis	2021-04-30 14:00:54 +02:00
Dominique Eifländer	184f55535a	Fixed dossier dictionary renalaysis	2021-04-30 13:52:46 +02:00
Dominique Eiflaender	979ab71090	Pull request #152 : Added isDossierDictionaryEntry to manualRedactions and redactionLog Merge in RED/redaction-service from isDossierDictionary to master * commit 'b7ae6cf390b1aa58c9f6210e3c8fcbfac89a6610': Added isDossierDictionaryEntry to manualRedactions and redactionLog	2021-04-29 12:36:09 +02:00
Dominique Eifländer	b7ae6cf390	Added isDossierDictionaryEntry to manualRedactions and redactionLog	2021-04-29 12:29:50 +02:00
Dominique Eiflaender	01eceaaef4	Pull request #151 : RED-1334: Enabled to use dictionaries per dossier Merge in RED/redaction-service from RED-1334 to master * commit '48beb984ca02ab163a64af38907a05d70b048655': RED-1334: Enabled to use dictionaries per dossier	2021-04-28 15:20:50 +02:00
Dominique Eifländer	48beb984ca	RED-1334: Enabled to use dictionaries per dossier	2021-04-28 15:11:11 +02:00
Timo Bejan	f7af37d287	Pull request #150 : cleaned up image code Merge in RED/redaction-service from image-cleanup to master * commit '1ddcafbdd23adb8354259660e25723449fb47855': cleaned up image code	2021-04-26 18:35:14 +02:00
Timo	1ddcafbdd2	cleaned up image code	2021-04-26 19:01:01 +03:00
Timo Bejan	e1b8ca4956	Pull request #149 : moved changelog to redaction service, removed filename from changelog Merge in RED/redaction-service from improved-redaction-performance to master * commit 'c05fdb48545b829ae82502c6d6ad486f2210f8f2': moved changelog to redaction service, removed filename from changelog	2021-04-26 14:52:44 +02:00
Timo	c05fdb4854	moved changelog to redaction service, removed filename from changelog	2021-04-26 15:48:17 +03:00
Clarissa Dietrich	e723208473	Pull request #148 : RED-1378: fix: merging images condition Merge in RED/redaction-service from mergingFix to master * commit '047ac9bf848d168b2a1bbe9217538b6bdac00de6': RED-1378: fix: merging images condition	2021-04-26 12:16:10 +02:00
Clarissa.Dietrich	047ac9bf84	RED-1378: fix: merging images condition	2021-04-26 10:50:35 +02:00
Timo Bejan	b83250f161	Pull request #141 : fix: merge images into one Merge in RED/redaction-service from classimg3 to master * commit '4d1ad3b5a56915e9e5df9191a535c0e5cd83972c': RED-1351: merge images into one	2021-04-23 09:36:04 +02:00
Clarissa.Dietrich	4d1ad3b5a5	RED-1351: merge images into one	2021-04-22 17:10:12 +02:00
Timo Bejan	24f33fe6bf	Pull request #147 : fixed changelog storage Merge in RED/redaction-service from improved-redaction-performance to master * commit '44ff9cb7757034a3df9c017f58a3381b52d73133': fixed changelog storage	2021-04-21 19:45:17 +02:00
Timo	44ff9cb775	fixed changelog storage	2021-04-21 20:41:18 +03:00
Timo Bejan	88524d0118	Pull request #146 : reworked api Merge in RED/redaction-service from redaction-system-queue-prep to master * commit '735cfdd6c4f8ffd6024f9803c7c5466205ebf7b4': reworked api	2021-04-21 17:57:05 +02:00
Timo	735cfdd6c4	reworked api	2021-04-21 18:53:29 +03:00
Timo Bejan	08f3ee8544	Pull request #145 : removed long test, fixed NPE Merge in RED/redaction-service from redaction-system-queue-prep to master * commit '3af121193214d250fe9dc58056f13a315d724184': fixed reanalyse condition fixed reanalyse condition removed long test, fixed NPE	2021-04-21 17:29:41 +02:00
Timo	3af1211932	fixed reanalyse condition	2021-04-21 18:16:39 +03:00
Timo	c1d5be8f63	fixed reanalyse condition	2021-04-21 18:13:35 +03:00
Timo	567bb00377	removed long test, fixed NPE	2021-04-21 18:13:35 +03:00
Timo Bejan	d7679c8c38	Pull request #144 : Updated code to respond to file-status service on successful/failed analysis Merge in RED/redaction-service from redaction-system-queue-prep to master * commit 'f6d0b2d0db867afcb8a3edfd48b1d101c681cd65': temporary ignore this added missing config yml Updated code to respond to file-status service on successful/failed analysis	2021-04-21 17:01:09 +02:00
Timo	f6d0b2d0db	temporary ignore this	2021-04-21 17:34:01 +03:00
Timo	43217e6abe	added missing config yml	2021-04-21 17:28:01 +03:00
Timo	8182b1a0fa	Updated code to respond to file-status service on successful/failed analysis	2021-04-21 17:15:49 +03:00
Timo Bejan	b22e3d93c0	Pull request #143 : prepared redaction service for queue handling Merge in RED/redaction-service from redaction-system-queue-prep to master * commit 'ab3b30b28256a93fff79b642080ddc1d940992a9': prepared redaction service for queue handling	2021-04-21 13:55:48 +02:00
Timo	ab3b30b282	prepared redaction service for queue handling	2021-04-21 14:11:22 +03:00
Timo	0d63f9372f	added reanalyse if possible flag to analyse request	2021-04-21 09:16:36 +03:00
Timo Bejan	07b05b2d89	Pull request #142 : Improved redaction performance Merge in RED/redaction-service from improved-redaction-performance to master * commit 'b34fc673c4d5a4440d6e5f2391db4420c4d2acf9': bamboo-specs/src/main/java/buildjob/PlanSpec.java edited online with Bitbucket fixed some test issues fixed pmd updated planspec updated redrect set mvn opts run tests with real life jvm args to detect oom issues early code format, dependecy and test update, logging for reanalysis reworked reanalysis and text storage Serialization of text	2021-04-20 10:30:15 +02:00
Timo Bejan	b34fc673c4	bamboo-specs/src/main/java/buildjob/PlanSpec.java edited online with Bitbucket	2021-04-20 09:57:25 +02:00
Timo	5b24c3a52c	fixed some test issues	2021-04-20 10:56:19 +03:00
Timo	385dff63ce	fixed pmd	2021-04-20 10:37:53 +03:00
Timo	f1ce52a19a	updated planspec	2021-04-20 10:33:30 +03:00
Timo	e48e4e1797	updated redrect	2021-04-20 10:32:54 +03:00
Timo	7af51f992a	set mvn opts	2021-04-20 10:32:11 +03:00
Timo	15f911ee73	run tests with real life jvm args to detect oom issues early	2021-04-20 10:28:40 +03:00
Timo	ba28a3e0d3	code format, dependecy and test update, logging for reanalysis	2021-04-20 10:26:27 +03:00
Timo	1d4708ad13	reworked reanalysis and text storage	2021-04-20 09:51:50 +03:00
Timo	5c2596e268	Serialization of text	2021-04-19 13:08:32 +03:00
Timo Bejan	674ebf8eb7	Pull request #140 : Improved redaction performance Merge in RED/redaction-service from improved-redaction-performance to master * commit '42fcea85d30f914fe02e304e69a551da409c6c3b': set image type on error proper error handling for image clasification	2021-04-18 11:39:27 +02:00
Timo	42fcea85d3	set image type on error	2021-04-18 11:31:33 +03:00
Timo	a34d2fb675	proper error handling for image clasification	2021-04-18 11:30:53 +03:00
Timo Bejan	e38aed22de	Pull request #139 : attempted fix for image clasification Merge in RED/redaction-service from improved-redaction-performance to master * commit 'ed5686dc51f57c9e772ccc3c0ef2a8955e7f3e73': Re-worked cache issues close final PDDocument fixed in-memory storage issues "fixed" memory issues by calling GC manually, removing soft reference cache and disposing images properly attempted fix for image clasification	2021-04-17 09:13:12 +02:00
Timo	ed5686dc51	Re-worked cache issues	2021-04-17 09:55:46 +03:00
Timo	8112f2035a	close final PDDocument	2021-04-17 09:18:01 +03:00
Timo	169ab20351	fixed in-memory storage issues	2021-04-17 09:10:40 +03:00
Timo	8060e3a29f	"fixed" memory issues by calling GC manually, removing soft reference cache and disposing images properly	2021-04-16 23:13:12 +03:00
Timo	4749858e80	attempted fix for image clasification	2021-04-16 20:53:09 +03:00
Timo Bejan	8429e262ea	Pull request #138 : added versions to analyze result Merge in RED/redaction-service from improved-redaction-performance to master * commit '93d75e2f1c9f692989b69d2c20df7073f6db45c7': added versions to analyze result	2021-04-16 15:12:39 +02:00
Timo	93d75e2f1c	added versions to analyze result	2021-04-16 15:52:17 +03:00
Timo Bejan	0ac55afbb0	Pull request #137 : Reworked re-analysis and analysis to use memory model / directly read/store files, and dumped pd doc wherever possible Merge in RED/redaction-service from improved-redaction-performance to master * commit '5cb4ea287cfecc61efbe959f182e9e7f4e2ba78c': Reworked re-analysis and analysis to use memory model / directly read/store files, and dumped pd doc wherever possible	2021-04-16 14:25:33 +02:00
Timo	5cb4ea287c	Reworked re-analysis and analysis to use memory model / directly read/store files, and dumped pd doc wherever possible	2021-04-16 14:50:04 +03:00
Dominique Eiflaender	ed59f36220	Pull request #136 : Fixed endless loop on documents that contains no text Merge in RED/redaction-service from EndlessLoopFix to master * commit 'c157a80630323aa0d654559636b7246e1901b598': Fixed endless loop on documents that contains no text	2021-04-16 10:12:55 +02:00