RED-3816: Bugfix with adding images to sections

Pull request #394 : RED-3800 String Performance matching test
Merge in RED/redaction-service from RED-3800-string-performance-test to master * commit '21d717f0837c9cc9b2f5b251d372db88b25a7b6d': RED-3800 String Performance matching test RED-3800 String Performance matching test
2022-05-30 13:52:57 +02:00 · 2022-05-24 11:09:08 +02:00 · 2022-05-24 12:01:10 +03:00 · 2022-05-24 11:58:34 +03:00 · 2022-05-24 09:00:07 +02:00 · 2022-05-23 18:48:10 +02:00
97 changed files with 49715 additions and 763 deletions
@@ -5,7 +5,7 @@
    <parent>
        <groupId>com.iqser.red</groupId>
        <artifactId>platform-docker-dependency</artifactId>
-        <version>1.1.0</version>
+        <version>1.2.0</version>
        <relativePath />
    </parent>
    <modelVersion>4.0.0</modelVersion>
@@ -42,7 +42,7 @@
                <artifactId>docker-maven-plugin</artifactId>
            </plugin>
        </plugins>
-        
+
        <pluginManagement>
            <plugins>
                <plugin>
@@ -95,4 +95,4 @@
            </plugins>
        </pluginManagement>
    </build>
-</project>
+</project>
@@ -23,20 +23,24 @@

    <properties>
        <pdfbox.version>2.0.24</pdfbox.version>
+        <dsljson.version>1.9.9</dsljson.version>
    </properties>


    <dependencyManagement>
-
        <dependencies>
+            <dependency>
+                <groupId>com.dslplatform</groupId>
+                <artifactId>dsl-json-java8</artifactId>
+                <version>${dsljson.version}</version>
+            </dependency>
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.11.0</version>
+                <version>1.13.0</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
-
            <dependency>
                <groupId>org.apache.pdfbox</groupId>
                <artifactId>pdfbox</artifactId>
@@ -47,9 +51,7 @@
                <artifactId>pdfbox-tools</artifactId>
                <version>${pdfbox.version}</version>
            </dependency>
-
        </dependencies>
-
    </dependencyManagement>

    <build>
@@ -58,12 +60,10 @@
                <plugin>
                    <groupId>org.sonarsource.scanner.maven</groupId>
                    <artifactId>sonar-maven-plugin</artifactId>
-                    <version>3.9.0.2155</version>
                    </plugin>
                <plugin>
                    <groupId>org.owasp</groupId>
                    <artifactId>dependency-check-maven</artifactId>
-                    <version>6.3.1</version>
                    <configuration>
                        <format>ALL</format>
                    </configuration>
@@ -88,27 +88,5 @@
                </plugin>
            </plugins>
        </pluginManagement>
-        <plugins>
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.8</version>
-                <executions>
-                    <execution>
-                        <id>prepare-agent</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>report</id>
-                        <goals>
-                            <goal>report-aggregate</goal>
-                        </goals>
-                        <phase>verify</phase>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
    </build>
 </project>
@@ -12,15 +12,24 @@
    <artifactId>redaction-service-api-v1</artifactId>

    <properties>
-        <persistence-service.version>1.143.0</persistence-service.version>
+        <persistence-service.version>1.166.0</persistence-service.version>
    </properties>

    <dependencies>
+
+        <!-- https://mvnrepository.com/artifact/com.dslplatform/dsl-json-java8 -->
+        <dependency>
+            <groupId>com.dslplatform</groupId>
+            <artifactId>dsl-json-java8</artifactId>
+            <version>${dsljson.version}</version>
+        </dependency>
+
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-web</artifactId>
            <optional>true</optional>
        </dependency>
+
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>persistence-service-api-v1</artifactId>
@@ -32,5 +41,21 @@
                </exclusion>
            </exclusions>
        </dependency>
+
    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <annotationProcessors>
+                        <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
+                        <annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
+                    </annotationProcessors>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
 </project>
@@ -1,18 +1,17 @@
 package com.iqser.red.service.redaction.v1.model;

+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
@Data
@Builder
@NoArgsConstructor
@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.model;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -1,12 +1,12 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.time.OffsetDateTime;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.time.OffsetDateTime;
+
@Data
@Builder
@AllArgsConstructor
@@ -1,13 +1,13 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.util.ArrayList;
-import java.util.List;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.util.ArrayList;
+import java.util.List;
+
@Data
@Builder
@NoArgsConstructor
@@ -1,14 +1,14 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
@Data
@Builder
@NoArgsConstructor
@@ -22,11 +22,7 @@ public class ManualChange {
    private OffsetDateTime processedDate;
    private OffsetDateTime requestedDate;
    private String userId;
-    private Map<String, Object> propertyChanges = new HashMap<>();
-
-    public boolean isProcessed() {
-        return processedDate != null;
-    }
+    private Map<String, String> propertyChanges = new HashMap<>();

    public static ManualChange from(BaseAnnotation baseAnnotation) {
        ManualChange manualChange = new ManualChange();
@@ -37,12 +33,16 @@ public class ManualChange {
        return manualChange;
    }

+    public boolean isProcessed() {
+        return processedDate != null;
+    }
+
    public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
        this.manualRedactionType = manualRedactionType;
        return this;
    }

-    public ManualChange withChange(String property, Object value) {
+    public ManualChange withChange(String property, String value) {
        this.propertyChanges.put(property, value);
        return this;
    }
@@ -1,14 +1,18 @@
 package com.iqser.red.service.redaction.v1.model;

-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.NoArgsConstructor;

+import java.util.ArrayList;
 import java.util.List;


@Data
+@CompiledJson
@AllArgsConstructor
+@NoArgsConstructor
 public class RedactionLog {


@@ -23,8 +27,8 @@ public class RedactionLog {
     */
    private int analysisNumber;

-    private List<RedactionLogEntry> redactionLogEntry;
-    private List<LegalBasis> legalBasis;
+    private List<RedactionLogEntry> redactionLogEntry = new ArrayList<>();
+    private List<RedactionLogLegalBasis> legalBasis = new ArrayList<>();

    private long dictionaryVersion = -1;
    private long dossierDictionaryVersion = -1;
@@ -0,0 +1,22 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.time.OffsetDateTime;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class RedactionLogComment {
+
+    private long id;
+    private String user;
+    private String text;
+    private String annotationId;
+    private String fileId;
+    private OffsetDateTime date;
+    private OffsetDateTime softDeletedTime;
+
+}
@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.model;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
 import lombok.*;

 import java.util.*;
@@ -41,7 +40,7 @@ public class RedactionLogEntry {
    private String textAfter;

    @Builder.Default
-    private List<Comment> comments = new ArrayList<>();
+    private List<RedactionLogComment> comments = new ArrayList<>();

    private int startOffset;
    private int endOffset;
@@ -54,6 +53,8 @@ public class RedactionLogEntry {

    private boolean excluded;

+    private String sourceId;
+
    @EqualsAndHashCode.Exclude
    @Builder.Default
    private List<Change> changes = new ArrayList<>();
@@ -0,0 +1,16 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class RedactionLogLegalBasis {
+
+    private String name;
+    private String description;
+    private String reason;
+
+}
@@ -1,20 +1,17 @@
 package com.iqser.red.service.redaction.v1.model;

-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
@Data
@Builder
@NoArgsConstructor
@@ -3,27 +3,16 @@ package com.iqser.red.service.redaction.v1.model;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;

@Data
-@RequiredArgsConstructor
@NoArgsConstructor
@AllArgsConstructor
 public class SectionArea {

-    @NonNull
    private Point topLeft;
-
-    @NonNull
    private float width;
-
-    @NonNull
    private float height;
-
-    @NonNull
    private int page;
-
    private String header;

    public boolean contains(Rectangle other) {
@@ -1,13 +1,14 @@
 package com.iqser.red.service.redaction.v1.model;

+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
-import lombok.RequiredArgsConstructor;

 import java.util.*;

@Data
+@CompiledJson
@AllArgsConstructor
@NoArgsConstructor
 public class SectionGrid {
@@ -17,13 +18,14 @@ public class SectionGrid {
    private List<SectionGridSection> sections = new ArrayList<>();

    @Data
-    @RequiredArgsConstructor
+    @NoArgsConstructor
+    @AllArgsConstructor
    public static class SectionGridSection {

-        private final int sectionNumber;
-        private final String headline;
-        private final Set<Integer> pages;
-        private final List<SectionArea> sectionAreas;
+        private int sectionNumber;
+        private String headline;
+        private Set<Integer> pages;
+        private List<SectionArea> sectionAreas;

    }
 }
@@ -3,30 +3,18 @@ package com.iqser.red.service.redaction.v1.model;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;

 import java.util.List;

@Data
@AllArgsConstructor
@NoArgsConstructor
-@RequiredArgsConstructor
 public class SectionRectangle {

-    @NonNull
    private Point topLeft;
-
-    @NonNull
    private float width;
-
-    @NonNull
    private float height;
-
-    @NonNull
    private int part;
-
-    @NonNull
    private int numberOfParts;

    private List<CellRectangle> tableCells;
@@ -1,20 +1,16 @@
 package com.iqser.red.service.redaction.v1.resources;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.*;
-
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestParam;

 public interface RedactionResource {

-    @PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
-
-
    @PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
    RedactionResult classify(@RequestBody RedactionRequest redactionRequest);

@@ -12,18 +12,44 @@
    <artifactId>redaction-service-server-v1</artifactId>

    <properties>
-        <drools.version>7.59.0.Final</drools.version>
-        <kie.version>7.59.0.Final</kie.version>
-        <locationtech.version>1.16.1</locationtech.version>
-        <pdfbox.jbig2-imageio.version>3.0.3</pdfbox.jbig2-imageio.version>
-        <jai-imageio.version>1.4.0</jai-imageio.version>
+        <drools.version>7.68.0.Final</drools.version>
+        <kie.version>7.68.0.Final</kie.version>
+        <locationtech.version>1.18.2</locationtech.version>
+        <javaassist.version>3.28.0-GA</javaassist.version>
+        <ahocorasick.version>0.6.3</ahocorasick.version>
+        <jackson.version>2.13.2</jackson.version>
    </properties>

    <dependencies>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-aop</artifactId>
+        </dependency>
+
        <dependency>
            <groupId>com.iqser.red.commons</groupId>
            <artifactId>storage-commons</artifactId>
        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-afterburner</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.ahocorasick</groupId>
+            <artifactId>ahocorasick</artifactId>
+            <version>${ahocorasick.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.javassist</groupId>
+            <artifactId>javassist</artifactId>
+            <version>${javaassist.version}</version>
+        </dependency>
+
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>redaction-service-api-v1</artifactId>
@@ -49,22 +75,6 @@
            <artifactId>guava</artifactId>
        </dependency>

-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>jbig2-imageio</artifactId>
-            <version>${pdfbox.jbig2-imageio.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.github.jai-imageio</groupId>
-            <artifactId>jai-imageio-core</artifactId>
-            <version>${jai-imageio.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.github.jai-imageio</groupId>
-            <artifactId>jai-imageio-jpeg2000</artifactId>
-            <version>${jai-imageio.version}</version>
-        </dependency>
-
        <!-- commons -->
        <dependency>
            <groupId>com.iqser.red.commons</groupId>
@@ -113,6 +123,18 @@

    <build>
        <plugins>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <annotationProcessors>
+                        <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
+                        <annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
+                    </annotationProcessors>
+                </configuration>
+            </plugin>
+
            <plugin>
                <!-- generate git.properties for exposure in /info -->
                <groupId>pl.project13.maven</groupId>
@@ -9,8 +9,12 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.openfeign.EnableFeignClients;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

+import io.micrometer.core.aop.TimedAspect;
+import io.micrometer.core.instrument.MeterRegistry;
+
@Import({DefaultWebMvcConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@@ -22,5 +26,10 @@ public class Application {
        SpringApplication.run(Application.class, args);
    }

+    @Bean
+    public TimedAspect timedAspect(MeterRegistry registry) {
+        return new TimedAspect(registry);
+    }
+

 }
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import lombok.AllArgsConstructor;
@@ -14,6 +15,7 @@ public class Footer {
    private List<TextBlock> textBlocks;

    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public SearchableText getSearchableText() {

        SearchableText searchableText = new SearchableText();
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import lombok.AllArgsConstructor;
@@ -14,6 +15,7 @@ public class Header {
    private List<TextBlock> textBlocks;

    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public SearchableText getSearchableText() {

        SearchableText searchableText = new SearchableText();
@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import com.dslplatform.json.CompiledJson;
+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
@@ -14,6 +16,7 @@ import java.util.*;

@Data
@Builder
+@CompiledJson
@NoArgsConstructor
@AllArgsConstructor
 public class SectionText {
@@ -38,6 +41,7 @@ public class SectionText {
    }

    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public SearchableText getSearchableText() {

        SearchableText searchableText = new SearchableText();
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -8,6 +9,7 @@ import java.util.ArrayList;
 import java.util.List;

@Data
+@CompiledJson
@NoArgsConstructor
@AllArgsConstructor
 public class Text {
@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import com.dslplatform.json.CompiledJson;
+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
@@ -15,6 +17,7 @@ import java.util.List;
@AllArgsConstructor
@Builder
@Data
+@CompiledJson
@NoArgsConstructor
 public class TextBlock extends AbstractTextContainer {

@@ -121,6 +124,7 @@ public class TextBlock extends AbstractTextContainer {

    @Override
    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getText() {

        StringBuilder sb = new StringBuilder();
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import lombok.AllArgsConstructor;
@@ -14,6 +15,7 @@ public class UnclassifiedText {
    private List<TextBlock> textBlocks;

    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public SearchableText getSearchableText() {

        SearchableText searchableText = new SearchableText();
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.client.model;

+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -7,6 +8,7 @@ import lombok.NoArgsConstructor;

@Data
@Builder
+@CompiledJson
@AllArgsConstructor
@NoArgsConstructor
 public class EntityRecogintionEntity {
@@ -4,18 +4,18 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;

+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

@Data
-@Builder
-@AllArgsConstructor
+@CompiledJson
@NoArgsConstructor
+@AllArgsConstructor
 public class NerEntities {

-   @Builder.Default
   private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();

 }
@@ -16,7 +16,6 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -33,42 +32,11 @@ public class RedactionController implements RedactionResource {

    private final PdfVisualisationService pdfVisualisationService;
    private final DroolsExecutionService droolsExecutionService;
-    private final DictionaryService dictionaryService;
-    private final AnnotationService annotationService;
    private final PdfSegmentationService pdfSegmentationService;
    private final RedactionStorageService redactionStorageService;
    private final RedactionLogMergeService redactionLogMergeService;
    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;

-
-    public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
-
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
-        var mergedRedactionLog = getRedactionLog(RedactionRequest.builder()
-                .fileId(annotateRequest.getFileId())
-                .manualRedactions(annotateRequest.getManualRedactions())
-                .dossierId(annotateRequest.getDossierId())
-                .dossierTemplateId(annotateRequest.getDossierTemplateId())
-                .build());
-        var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
-
-        try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
-            pdDocument.setAllSecurityToBeRemoved(true);
-
-            dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
-            annotationService.annotate(pdDocument, mergedRedactionLog, sectionsGrid);
-
-            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
-                pdDocument.save(byteArrayOutputStream);
-                return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
-            }
-
-        } catch (Exception e) {
-            throw new RedactionException(e);
-        }
-    }
-
-
    @Override
    public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {

@@ -157,39 +125,8 @@ public class RedactionController implements RedactionResource {
    @Override
    public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {

-        log.debug("Requested preview for: {}", redactionRequest);

-        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-        if (redactionLog == null) {
-            throw new NotFoundException("RedactionLog not present");
-        }
-
-        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-
-        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
-        if (sectionGrid.getSections().isEmpty()) {
-
-            log.info("SectionGrid does not have headlines set. Computing headlines now!");
-            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-            // enhance section grid with headline data
-            for (var sectionText : text.getSectionTexts()) {
-                sectionGrid.getSections()
-                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText.getSectionAreas()
-                                .stream()
-                                .map(SectionArea::getPage)
-                                .collect(Collectors.toSet()), sectionText.getSectionAreas()));
-            }
-            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
-        }
-
-        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        var merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
-
-        merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
-
-        return merged;
+        return redactionLogMergeService.provideRedactionLog(redactionRequest);
    }


@@ -1,52 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.memory;
-
-import lombok.extern.slf4j.Slf4j;
-
-import java.text.CharacterIterator;
-import java.text.StringCharacterIterator;
-
-@Slf4j
-public class MemoryStats {
-
-
-    public static void printMemoryStats() {
-        log.info("\n\n ------------------------------ \n" +
-                " Used Memory: " + humanReadableByteCountBin(getUsedMemory()) + "\n" +
-                " Free Memory: " + humanReadableByteCountBin(getFreeMemory()) + "\n" +
-                " Total Memory: " + humanReadableByteCountBin(getTotalMemory()) + "\n" +
-                " Max Memory: " + humanReadableByteCountBin(getMaxMemory()) + "\n" +
-                "\n ------------------------------ \n");
-    }
-
-
-    public static String humanReadableByteCountBin(long bytes) {
-        long absB = bytes == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(bytes);
-        if (absB < 1024) {
-            return bytes + " B";
-        }
-        long value = absB;
-        CharacterIterator ci = new StringCharacterIterator("KMGTPE");
-        for (int i = 40; i >= 0 && absB > 0xfffccccccccccccL >> i; i -= 10) {
-            value >>= 10;
-            ci.next();
-        }
-        value *= Long.signum(bytes);
-        return String.format("%.1f %ciB", value / 1024.0, ci.current());
-    }
-
-    private static long getMaxMemory() {
-        return Runtime.getRuntime().maxMemory();
-    }
-
-    private static long getUsedMemory() {
-        return getMaxMemory() - getFreeMemory();
-    }
-
-    private static long getTotalMemory() {
-        return Runtime.getRuntime().totalMemory();
-    }
-
-    private static long getFreeMemory() {
-        return Runtime.getRuntime().freeMemory();
-    }
-}
@@ -1,52 +1,26 @@
 package com.iqser.red.service.redaction.v1.server.parsing;

-import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.reflect.FieldUtils;
-import org.apache.pdfbox.contentstream.operator.Operator;
-import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
-import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
-import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
-import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
-import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
-import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
-import org.apache.pdfbox.cos.COSBase;
-import org.apache.pdfbox.cos.COSName;
-import org.apache.pdfbox.cos.COSNumber;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.graphics.PDXObject;
-import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
-import org.apache.pdfbox.text.TextPosition;
-import org.apache.pdfbox.util.Matrix;
-
 import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
-
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pdfbox.contentstream.operator.Operator;
+import org.apache.pdfbox.contentstream.operator.OperatorName;
+import org.apache.pdfbox.contentstream.operator.color.*;
+import org.apache.pdfbox.contentstream.operator.state.*;
+import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
+import org.apache.pdfbox.cos.COSBase;
+import org.apache.pdfbox.cos.COSNumber;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.text.TextPosition;
+
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;

@Slf4j
 public class PDFLinesTextStripper extends PDFTextStripper {
@@ -66,8 +40,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private int minCharHeight;
    @Getter
    private int maxCharHeight;
-    @Getter
-    private List<PdfImage> images = new ArrayList<>();

    private float path_x;
    private float path_y;
@@ -183,9 +155,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                graphicsPath.clear();
                break;

-//            case OperatorName.DRAW_OBJECT:
-//                processImageOperation(arguments);
-//                break;

        }

@@ -193,32 +162,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    }


-    protected void processImageOperation(List<COSBase> arguments) {
-
-        try {
-            COSName objectName = (COSName) arguments.get(0);
-            PDXObject xobject = getResources().getXObject(objectName);
-            if (xobject instanceof PDImageXObject) {
-                PDImageXObject image = (PDImageXObject) xobject;
-                Matrix ctmNew = getGraphicsState().getCurrentTransformationMatrix();
-
-                Rectangle2D rect = new Rectangle2D.Float(ctmNew.getTranslateX(), ctmNew.getTranslateY(), ctmNew.getScaleX(), ctmNew.getScaleY());
-
-                // Memory Hack - sofReference kills me
-                FieldUtils.writeField(image, "cachedImageSubsampling", -1, true);
-
-                if (rect.getHeight() > 2 && rect.getWidth() > 2) {
-                    this.images.add(new PdfImage(image.getImage(), rect, pageNumber, image.getImage()
-                            .getColorModel()
-                            .hasAlpha()));
-                }
-            }
-        } catch (Exception e) {
-            log.warn("Problem during image extraction: {}", e.getMessage());
-        }
-    }
-
-
    private float floatValue(COSBase value) {

        if (value instanceof COSNumber) {
@@ -375,7 +318,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        minCharHeight = Integer.MAX_VALUE;
        maxCharHeight = 0;
        textPositionSequences.clear();
-        images = new ArrayList<>();
        rulings.clear();
        graphicsPath.clear();
        path_x = 0.0f;
@@ -1,14 +1,23 @@
 package com.iqser.red.service.redaction.v1.server.parsing.model;

-import com.fasterxml.jackson.annotation.JsonIgnore;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.SneakyThrows;
 import org.apache.pdfbox.text.TextPosition;
 import org.springframework.beans.BeanUtils;

+import com.dslplatform.json.CompiledJson;
+import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.SneakyThrows;
+
@Data
+@Builder
@NoArgsConstructor
+@AllArgsConstructor
+@CompiledJson
 public class RedTextPosition {

    private String textMatrix;
@@ -26,14 +35,17 @@ public class RedTextPosition {

    // not used in reanalysis
    @JsonIgnore
+    @JsonAttribute(ignore = true)
    private float widthOfSpace;

    // not used in reanalysis
    @JsonIgnore
+    @JsonAttribute(ignore = true)
    private float fontSizeInPt;

    // not used in reanalysis
    @JsonIgnore
+    @JsonAttribute(ignore = true)
    private String fontName;


@@ -6,18 +6,25 @@ import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;

+import com.dslplatform.json.CompiledJson;
+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;

+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@Data
+@Builder
+@CompiledJson
@NoArgsConstructor
+@AllArgsConstructor
@JsonIgnoreProperties({"empty"})
 public class TextPositionSequence implements CharSequence {

@@ -27,7 +34,6 @@ public class TextPositionSequence implements CharSequence {
    private float x1;
    private float x2;

-
    public TextPositionSequence(int page) {

        this.page = page;
@@ -112,6 +118,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getX1() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -123,6 +130,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getX2() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -135,6 +143,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getRotationAdjustedY() {

        return textPositions.get(0).getY();
@@ -142,6 +151,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getRotationAdjustedX() {

        return textPositions.get(0).getXDirAdj();
@@ -149,6 +159,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getY1() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -160,6 +171,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getY2() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -171,6 +183,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + 2;
@@ -178,6 +191,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getHeight() {

        return getY2() - getY1();
@@ -185,6 +199,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getWidth() {

        return getX2() - getX1();
@@ -192,6 +207,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getFont() {

        return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
@@ -199,6 +215,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getFontStyle() {

        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();
@@ -217,6 +234,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getFontSize() {

        return textPositions.get(0).getFontSizeInPt();
@@ -224,6 +242,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getSpaceWidth() {

        return textPositions.get(0).getWidthOfSpace();
@@ -231,6 +250,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public int getRotation() {

        return textPositions.get(0).getRotation();
@@ -238,6 +258,7 @@ public class TextPositionSequence implements CharSequence {


    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public Rectangle getRectangle() {

        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), textPositions.get(0).getRotation(), textPositions.get(0).getDir());
@@ -292,11 +313,11 @@ public class TextPositionSequence implements CharSequence {
            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;

        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
-            posXInit = textPositions.get(0).getPageWidth() - getX1() +1;
+            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
                    .getWidthDirAdj() - 3;
            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
-            posYEnd =  textPositions.get(textPositions.size() - 1)
+            posYEnd = textPositions.get(textPositions.size() - 1)
                    .getYDirAdj() - height + 2;

        } else {
@@ -54,16 +54,6 @@ public class Dictionary {
    }


-    public boolean containsValue(String type, String value) {
-
-        return localAccessMap.containsKey(type) && localAccessMap.get(type)
-                .getValues(false)
-                .contains(value) || localAccessMap.containsKey(type) && localAccessMap.get(type)
-                .getValues(true)
-                .contains(value);
-    }
-
-
    public boolean isHint(String type) {

        DictionaryModel model = localAccessMap.get(type);
@@ -23,4 +23,4 @@ public class DictionaryEntries {
    @Builder.Default
    Set<DictionaryEntry> falseRecommendations = new HashSet<>();

-}
+}
@@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;


 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import lombok.AllArgsConstructor;
 import lombok.Data;

 import java.io.Serializable;
+import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;

@@ -14,30 +16,76 @@ import java.util.stream.Collectors;
@AllArgsConstructor
 public class DictionaryModel implements Serializable {

-    private String type;
-    private int rank;
-    private float[] color;
-    private boolean caseInsensitive;
-    private boolean hint;
-    private Set<DictionaryEntry> entries;
-    private Set<DictionaryEntry> falsePositives;
-    private Set<DictionaryEntry> falseRecommendations;
-    private Set<String> localEntries;
-    private boolean isDossierDictionary;
+    private final String type;
+    private final int rank;
+    private final float[] color;
+    private final boolean caseInsensitive;
+    private final boolean hint;
+    private final boolean isDossierDictionary;
+    private final Set<DictionaryEntry> entries;
+    private final Set<DictionaryEntry> falsePositives;
+    private final Set<DictionaryEntry> falseRecommendations;
+
+    private transient SearchImplementation entriesSearch;
+    private transient SearchImplementation falsePositiveSearch;
+    private transient SearchImplementation falseRecommendationsSearch;
+    private transient SearchImplementation localSearch;
+
+    private final Set<String> localEntries = new HashSet<>();
+
+    public DictionaryModel(String type,
+                           int rank,
+                           float[] color,
+                           boolean caseInsensitive,
+                           boolean hint,
+                           Set<DictionaryEntry> entries,
+                           Set<DictionaryEntry> falsePositives,
+                           Set<DictionaryEntry> falseRecommendations,
+                           boolean isDossierDictionary) {
+        this.type = type;
+        this.rank = rank;
+        this.color = color;
+        this.caseInsensitive = caseInsensitive;
+        this.hint = hint;
+        this.isDossierDictionary = isDossierDictionary;
+
+        this.entries = entries;
+        this.falsePositives = falsePositives;
+        this.falseRecommendations = falseRecommendations;
+
+        this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);

-    public Set<String> getValues(boolean local) {
-        return local ? localEntries : entries.stream().filter(e -> !e.isDeleted()).map(e -> e.getValue()).collect(Collectors
-                .toSet());
    }

-    public Set<String> getFalsePositiveValues() {
-        return falsePositives.stream().filter(e -> !e.isDeleted()).map(e -> e.getValue()).collect(Collectors
-                .toSet());
+    public SearchImplementation getLocalSearch() {
+        if (this.localSearch == null) {
+            this.localSearch = new SearchImplementation(this.localEntries, caseInsensitive);
+        }
+        return this.localSearch;
    }

-    public Set<String> getFalseRecommendationValues() {
-        return falseRecommendations.stream().filter(e -> !e.isDeleted()).map(e -> e.getValue()).collect(Collectors
-                .toSet());
+    public SearchImplementation getEntriesSearch() {
+        if (entriesSearch == null) {
+            this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        }
+        return entriesSearch;
    }

+    public SearchImplementation getFalsePositiveSearch() {
+        if (falsePositiveSearch == null) {
+            this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        }
+        return falsePositiveSearch;
+    }
+
+    public SearchImplementation getFalseRecommendationsSearch() {
+        if (falseRecommendationsSearch == null) {
+            this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        }
+        return falseRecommendationsSearch;
+    }
+
+
 }
@@ -1,17 +1,23 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity implements ReasonHolder {

@@ -1,15 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
-
 import java.util.ArrayList;
 import java.util.List;

+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
+
@Data
+@Builder
@RequiredArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -7,6 +8,7 @@ import lombok.NoArgsConstructor;

@Data
@Builder
+@CompiledJson
@NoArgsConstructor
@AllArgsConstructor
 public class Image implements ReasonHolder {
@@ -1,34 +1,20 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;

-import java.awt.geom.Rectangle2D;
-import java.awt.image.BufferedImage;
-
@Data
@RequiredArgsConstructor
 public class PdfImage {

-    @JsonIgnore
-    private BufferedImage image;
    @NonNull
    private RedRectangle2D position;
    @NonNull
    private ImageType imageType;
    private boolean isAppendedToParagraph;
    private boolean hasTransparency;
-
    @NonNull
    private int page;

-    public PdfImage(BufferedImage image, Rectangle2D position, int page, boolean hasTransparency) {
-        this.image = image;
-        this.position = new RedRectangle2D(position.getX(), position.getY(), position.getWidth(), position.getHeight());
-        this.page = page;
-        this.hasTransparency = hasTransparency;
-    }
-
 }
@@ -1,12 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-
+import com.dslplatform.json.CompiledJson;
+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

@Data
+@Builder
+@CompiledJson
@NoArgsConstructor
@AllArgsConstructor
 public class RedRectangle2D {
@@ -19,6 +24,7 @@ public class RedRectangle2D {
    private double height;

    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public boolean isEmpty() {
        return width <= 0.0f || height <= 0.0f;
    }
@@ -1,16 +1,22 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;

 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.regex.Pattern;

 public class SearchableText {

+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    private transient String stringRepresentation;
    private final List<TextPositionSequence> sequences = new ArrayList<>();


@@ -88,8 +94,8 @@ public class SearchableText {
                        .get(i)
                        .charAt(j, caseInsensitive) == '-') {

-                    if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
-                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(searchSpace.get(i - 1)
+                    if (counter != 0 || i == 0 && j == 0 || j != 0 && SeparatorUtils.isSeparator(searchSpace.get(i)
+                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && SeparatorUtils.isSeparator(searchSpace.get(i - 1)
                            .charAt(searchSpace.get(i - 1)
                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
                            .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i)
@@ -105,9 +111,9 @@ public class SearchableText {
                        crossSequenceParts.add(partMatch);

                        if (i == searchSpace.size() - 1 && j == searchSpace.get(i)
-                                .length() - 1 || j != searchSpace.get(i).length() - 1 && isSeparator(searchSpace.get(i)
+                                .length() - 1 || j != searchSpace.get(i).length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i)
                                .charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
-                                .length() - 1 && isSeparator(searchSpace.get(i + 1)
+                                .length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i + 1)
                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i)
                                .length() - 1 && searchSpace.get(i)
                                .charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
@@ -175,18 +181,18 @@ public class SearchableText {
    }


-    private boolean isSeparator(char c) {
-
-        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
-    }
-
-
    @Override
    public String toString() {
-
        return buildString(sequences);
    }

+    public String asString() {
+        if (stringRepresentation == null) {
+            stringRepresentation = buildString(sequences);
+        }
+        return stringRepresentation;
+    }
+

    public String buildString(List<TextPositionSequence> sequences) {

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
@@ -22,9 +21,12 @@ import org.apache.commons.lang3.StringUtils;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;

 import lombok.Builder;
 import lombok.Data;
@@ -68,12 +70,17 @@ public class Section {
    @Builder.Default
    private List<FileAttribute> fileAttributes = new ArrayList<>();

+    @Builder.Default
+    private List<SectionArea> sectionAreas = new ArrayList<>();

-    public void addAiEntities(String type, String asType) {
+
+    @SuppressWarnings("unused")
+    @WhenCondition
+    public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {

        Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
-        Set<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toSet());
-        Set<Entity> found = EntitySearchUtils.findEntities(searchText, values, dictionary.getType(asType), headline, sectionNumber, false, false, Engine.NER, true, true);
+        List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
+        Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);

        Set<Entity> finalResult = new HashSet<>();
@@ -98,8 +105,11 @@ public class Section {
        nerEntities.removeAll(entitiesOfType);
    }

-
-    public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType, int minPartMatches, boolean allowDuplicateTypes) {
+    @SuppressWarnings("unused")
+    @WhenCondition
+    public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType, @Argument(ArgumentType.TYPE) String combineTypes,
+                               @Argument(ArgumentType.INTEGER) int maxDistanceBetween, @Argument(ArgumentType.TYPE) String asType,
+                               @Argument(ArgumentType.INTEGER) int minPartMatches, @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) {

        Set<String> combineSet = Set.of(combineTypes.split(","));

@@ -160,35 +170,35 @@ public class Section {
        }
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {

        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) {

        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {

        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {

        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) {

@@ -196,14 +206,14 @@ public class Section {
                .anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue()));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {

        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) {

@@ -211,37 +221,37 @@ public class Section {
        return tabularData != null && tabularData.containsKey(cleanHeaderName);
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean aiMatchesType(@Argument(ArgumentType.TYPE) String type) {

        return nerEntities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {

        return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {

        return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type));
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) {

        return StringUtils.containsIgnoreCase(headline, word);
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
-    public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive){
+    public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) {

        var compiledPattern = Patterns.getCompiledPattern(regEx, patternCaseInsensitive);

@@ -250,7 +260,7 @@ public class Section {
        return matcher.find();
    }

-
+    @SuppressWarnings("unused")
    @WhenCondition
    public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) {

@@ -259,18 +269,20 @@ public class Section {
        return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value);
    }

+
    @ThenAction
-    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
-                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
+    @SuppressWarnings("unused")
+    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {

        expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null);
    }


    @ThenAction
+    @SuppressWarnings("unused")
    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
-                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
-                              @Argument(ArgumentType.REGEX) String valuePattern) {
+                                    @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
+                                    @Argument(ArgumentType.REGEX) String valuePattern) {

        if (StringUtils.isEmpty(prefixPattern)) return;

@@ -279,6 +291,7 @@ public class Section {

        Set<Entity> expanded = new HashSet<>();
        for (var entity : entities) {
+            System.out.println(entity.getWord());

            if (!entity.getType().equals(type) || entity.getTextBefore() == null) {
                continue;
@@ -315,6 +328,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {

@@ -323,6 +337,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
                              @Argument(ArgumentType.REGEX) String valuePattern) {
@@ -370,6 +385,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
                            @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {

@@ -378,6 +394,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {

        redactImage(type, ruleNumber, reason, null, false);
@@ -385,6 +402,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
                       @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {

@@ -393,6 +411,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {

        redact(type, ruleNumber, reason, null, false);
@@ -400,6 +419,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason,
                                @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -409,6 +429,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                   @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) {

@@ -418,6 +439,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                              @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                              @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -427,6 +449,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                                 @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                 @Argument(ArgumentType.STRING) String reason) {
@@ -436,6 +459,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                              @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -445,6 +469,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                 @Argument(ArgumentType.STRING) String reason) {
@@ -454,6 +479,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
                                   @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                   @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -463,6 +489,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                      @Argument(ArgumentType.STRING) String reason) {
@@ -472,6 +499,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
                           @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason,
                           @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -481,6 +509,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
                              @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason) {

@@ -489,6 +518,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                                          @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                          @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -498,6 +528,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                                             @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
                                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
@@ -507,6 +538,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                                         @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) {

@@ -524,6 +556,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {

@@ -542,6 +575,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                   @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {

@@ -557,6 +591,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                             @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {

@@ -564,22 +599,25 @@ public class Section {
        EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
    }

-
-    public void ignore(String type) {
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void ignore(@Argument(ArgumentType.TYPE) String type) {

        entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.ENTITY));
    }

-
-    public void ignoreRecommendations(String type) {
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) {

        entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.RECOMMENDATION));
    }


    @ThenAction
+    @SuppressWarnings("unused")
    public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern,
-                                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
+                                             @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {

        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);

@@ -607,6 +645,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                                         @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) {

@@ -625,6 +664,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) {

        Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE, false);
@@ -633,6 +673,7 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) {

        String[] values = StringUtils.substringsBetween(text, start, "\n");
@@ -657,12 +698,48 @@ public class Section {


    @ThenAction
+    @SuppressWarnings("unused")
    public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) {

        annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
    }


+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        for (SectionArea sectionArea : sectionAreas) {
+
+            RedRectangle2D position = RedRectangle2D.builder()
+                    .height(sectionArea.getHeight() + 4)
+                    .width(sectionArea.getWidth() + 4)
+                    .x(sectionArea.getTopLeft().getX() - 2)
+                    .y(sectionArea.getTopLeft().getY() - 2)
+                    .build();
+
+            log.debug("SectionArea: {}", sectionArea);
+            log.debug("Position {}", position.toString());
+
+            Image image = Image.builder()
+                    .page(sectionArea.getPage())
+                    .position(position)
+                    .redaction(true)
+                    .hasTransparency(false)
+                    .sectionNumber(sectionNumber)
+                    .section(headline)
+                    .matchedRule(ruleNumber)
+                    .legalBasis(legalBasis)
+                    .redactionReason(reason)
+                    .type(type)
+                    .build();
+
+            images.add(image);
+        }
+    }
+
+
    private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis,
                                           boolean redaction) {

@@ -682,10 +759,8 @@ public class Section {
    private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine, boolean asRecommendation) {

        String text = caseInsensitive ? searchText.toLowerCase() : searchText;
-        String searchValue = caseInsensitive ? value.toLowerCase() : value;
-
-        Set<Entity> found = EntitySearchUtils.findEntities(text, Set.of(searchValue), dictionary.getType(asType), headline, sectionNumber, false, false, engine, false, asRecommendation);
-
+        Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType),
+                new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
        found.forEach(entity -> {
            if (redacted) {
                entity.setRedaction(true);
@@ -3,9 +3,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 import java.util.HashMap;
 import java.util.Map;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class Classification {

    private Map<String, Float> probabilities = new HashMap<>();
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class FilterGeometry {

    private ImageSize imageSize;
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class Filters {

    private FilterGeometry geometry;
@@ -1,9 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class Geometry {
    private float width;
    private float height;
-}
+}
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class ImageFormat {

    private float quotient;
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class ImageMetadata {

    private Classification classification;
@@ -1,14 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
+import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
 import java.util.ArrayList;
 import java.util.List;

-import com.fasterxml.jackson.annotation.JsonAlias;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.Data;
-
@Data
+@CompiledJson
 public class ImageServiceResponse {

    private String dossierId;
@@ -16,6 +18,7 @@ public class ImageServiceResponse {

    @JsonProperty(value = "imageMetadata")
    @JsonAlias("data")
+    @JsonAttribute(alternativeNames = {"imageMetadata"})
    private List<ImageMetadata> data = new ArrayList<>();

 }
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class ImageSize {

    private float quotient;
@@ -1,12 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class Position {
    private float x1;
    private float x2;
    private float y1;
    private float y2;
    private int pageNumber;
-}
+}
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;

+import com.dslplatform.json.CompiledJson;
 import lombok.Data;

@Data
+@CompiledJson
 public class Probability {

    private boolean unconfident;
@@ -1,33 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-import org.springframework.web.bind.annotation.RequestBody;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
+import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
@@ -35,21 +15,27 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;

+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;

@Slf4j
@Service
@@ -71,6 +57,7 @@ public class AnalyzeService {
    private final ImportedRedactionService importedRedactionService;


+    @Timed("analyzeDocumentStructure")
    public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {

        long startTime = System.currentTimeMillis();
@@ -92,6 +79,7 @@ public class AnalyzeService {
        }

        List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
+
        sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);

        Text text = new Text(pageCount, sectionTexts);
@@ -118,6 +106,7 @@ public class AnalyzeService {
    }


+    @Timed("reanalyze")
    @SneakyThrows
    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {

@@ -131,10 +120,14 @@ public class AnalyzeService {
            return analyze(analyzeRequest);
        }

+//        var dis = System.currentTimeMillis();
        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
+//        log.info("Dictionary Increment time time: {} ms", (System.currentTimeMillis() - dis));

+//        var fis = System.currentTimeMillis();
        Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse()
                .isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest);
+//        log.info("Find sections time: {} ms", (System.currentTimeMillis() - fis));

        if (sectionsToReanalyse.isEmpty()) {
            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
@@ -144,7 +137,7 @@ public class AnalyzeService {
        if (redactionServiceSettings.isNerServiceEnabled()) {
            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
        } else {
-            nerEntities = NerEntities.builder().build();
+            nerEntities = new NerEntities();
        }

        List<SectionText> reanalysisSections = text.getSectionTexts()
@@ -152,24 +145,36 @@ public class AnalyzeService {
                .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
                .collect(Collectors.toList());

+//        long kis = System.currentTimeMillis();
        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
+//        log.info("Kie time: {} ms", (System.currentTimeMillis() - kis));

+//        long dds = System.currentTimeMillis();
        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+//        log.info("Dict Time time: {} ms", (System.currentTimeMillis() - dds));

+//        long pis = System.currentTimeMillis();
        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
+//        log.info("Find Entities time: {}", (System.currentTimeMillis() - pis));
+
+//        long crs = System.currentTimeMillis();
        var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
-
+//        log.info("Create Redaction-log time: {} ms", (System.currentTimeMillis() - crs));
+//        long prs = System.currentTimeMillis();
        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newRedactionLogEntries, false);
-
+//        log.info("Process imports time: {} ms", (System.currentTimeMillis() - prs));
        redactionLog.getRedactionLogEntry()
                .removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType()
                        .equals(IMPORTED_REDACTION_TYPE));
        redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);

+//        var fls = System.currentTimeMillis();
+        //        log.info("Finalize time: {} ms", (System.currentTimeMillis() - fls));
        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
    }


+    @Timed("analyze")
    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {

        long startTime = System.currentTimeMillis();
@@ -179,7 +184,7 @@ public class AnalyzeService {
        if (redactionServiceSettings.isNerServiceEnabled()) {
            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
        } else {
-            nerEntities = NerEntities.builder().build();
+            nerEntities = new NerEntities();
        }
        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
@@ -191,7 +196,7 @@ public class AnalyzeService {
        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());

        var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, legalBasis, dictionary.getVersion()
+        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, convert(legalBasis), dictionary.getVersion()
                .getDossierTemplateVersion(), dictionary.getVersion()
                .getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));

@@ -219,13 +224,19 @@ public class AnalyzeService {
            }
        }

+//        long ss = System.currentTimeMillis();
+
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream()
+                .map(DictionaryIncrementValue::getValue).collect(Collectors.toList()), true);
+
        for (SectionText sectionText : text.getSectionTexts()) {

-            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrement.getValues())) {
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
                sectionsToReanalyse.add(sectionText.getSectionNumber());
            }

        }
+//        log.info("Section Find time: {}", (System.currentTimeMillis() - ss));

        log.info("Should reanalyze {} sections for request: {}, took: {}", sectionsToReanalyse.size(), analyzeRequest, System.currentTimeMillis() - start);

@@ -282,6 +293,9 @@ public class AnalyzeService {
                .map(ManualForceRedaction::getAnnotationId)))).collect(Collectors.toSet());
    }

+    public List<RedactionLogLegalBasis> convert(List<LegalBasis> legalBasis) {
+        return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
+    }

    public Image convert(RedactionLogEntry entry) {

@@ -125,10 +125,14 @@ public class DictionaryService {

                            DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);

-                            var newValues = newEntries.getEntries().stream().map(v -> v.getValue()).collect(Collectors.toSet());
-                            var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(v -> v.getValue()).collect(Collectors.toSet());
-                            var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(v -> v.getValue()).collect(Collectors.toSet());
+                            var newValues = newEntries.getEntries().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
+                            var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
+                            var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());

+
+                            oldModel.ifPresent(oldDictionaryModel -> {
+
+                            });
                            // add old entries from existing DictionaryModel
                            oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries().stream().filter(
                                    f -> !newValues.contains(f.getValue())).collect(Collectors.toList())
@@ -146,7 +150,7 @@ public class DictionaryService {
                            falseRecommendations.addAll(newEntries.getFalseRecommendations());

                            return new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
-                                    .isHint(), entries, falsePositives, falseRecommendations, new HashSet<>(), dossierId != null);
+                                    .isHint(), entries, falsePositives, falseRecommendations, dossierId != null);
                        })
                        .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                        .collect(Collectors.toList());
@@ -193,6 +197,7 @@ public class DictionaryService {
    }


+
    private float[] convertColor(String hex) {

        Color color = Color.decode(hex);
@@ -200,16 +205,6 @@ public class DictionaryService {
    }


-    public boolean isCaseInsensitiveDictionary(String type, String dossierTemplateId) {
-
-        DictionaryModel dictionaryModel = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
-        if (dictionaryModel != null) {
-            return dictionaryModel.isCaseInsensitive();
-        }
-        return false;
-    }
-
-
    public float[] getColor(String type, String dossierTemplateId) {

        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
@@ -253,23 +248,12 @@ public class DictionaryService {
    }


-    public float[] getRequestRemoveColor(String dossierTemplateId) {
-
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestRemoveColor();
-    }
-
-
    public float[] getNotRedactedColor(String dossierTemplateId) {

        return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
    }


-    public float[] getRequestAddColor(String dossierTemplateId) {
-
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
-    }
-
    private Long getVersion(DictionaryRepresentation dictionaryRepresentation) {
        if (dictionaryRepresentation == null) {
            return null;
@@ -32,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

@@ -86,6 +87,7 @@ public class EntityRedactionService {
                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
            }

+
            if (!local && analyzeRequest.getManualRedactions() != null) {

                var approvedForceRedactions = analyzeRequest.getManualRedactions().getForceRedactions().stream()
@@ -96,9 +98,10 @@ public class EntityRedactionService {
                var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
                        .filter(idr -> idr.getRequestDate() != null)
-                        .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
+                        .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
                        .map(IdRemoval::getAnnotationId).collect(Collectors.toSet());

+
                if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
                        .isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
                    for (Image image : reanalysisSection.getImages()) {
@@ -124,13 +127,14 @@ public class EntityRedactionService {
                }));
            }

+            log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
+
            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                    .isLocal(false)
                    .dictionaryTypes(dictionary.getTypes())
                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream
                            .concat(entities.getEntities().stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber())
-                                    .stream())
-                            .collect(Collectors.toSet()) : entities.getEntities())
+                                    .stream()).collect(Collectors.toSet()) : entities.getEntities())
                    .nerEntities(entities.getNerEntities())
                    .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
                    .searchText(reanalysisSection.getSearchableText().toString())
@@ -140,8 +144,10 @@ public class EntityRedactionService {
                    .searchableText(reanalysisSection.getSearchableText())
                    .dictionary(dictionary)
                    .images(reanalysisSection.getImages())
+                    .sectionAreas(reanalysisSection.getSectionAreas())
                    .fileAttributes(analyzeRequest.getFileAttributes())
                    .build(), reanalysisSection.getSearchableText()));
+
        }

        Set<Entity> entities = new HashSet<>();
@@ -200,18 +206,18 @@ public class EntityRedactionService {
    private void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) {

        analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> {
-                analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
+            analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {

-                    if (dictionary.getLocalAccessMap().get(key) == null) {
-                        log.warn("Dictionary {} is null", key);
-                    }
+                if (dictionary.getLocalAccessMap().get(key) == null) {
+                    log.warn("Dictionary {} is null", key);
+                }

-                    if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
-                        log.warn("Dictionary {} localEntries is null", key);
-                    }
+                if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
+                    log.warn("Dictionary {} localEntries is null", key);
+                }

-                    dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
-                });
+                dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
+            });
        });
    }

@@ -221,18 +227,20 @@ public class EntityRedactionService {
                                  List<Integer> cellStarts) {

        Set<Entity> found = new HashSet<>();
-        String searchableString = searchableText.toString();
+        String searchableString = searchableText.asString();
+
        if (StringUtils.isEmpty(searchableString)) {
            return new Entities(new HashSet<>(), new HashSet<>());
        }

        String lowercaseInputString = searchableString.toLowerCase();
        for (DictionaryModel model : dictionary.getDictionaryModels()) {
-            if (model.isCaseInsensitive()) {
-                EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.findEntities(lowercaseInputString, model.getValues(local), model, headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, false, local ? true : false));
-            } else {
-                EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.findEntities(searchableString, model.getValues(local), model, headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, false, local ? true : false));
-            }
+
+            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation, model, new FindEntityDetails(model.getType(),headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, local? EntityType.RECOMMENDATION: EntityType.ENTITY));
+
+            EntitySearchUtils.addOrAddEngine(found, entities);
        }

        Set<Entity> nerFound = new HashSet<>();
@@ -1,12 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
@@ -21,10 +14,17 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;

@Slf4j
@Service
@@ -88,10 +88,10 @@ public class ManualRedactionSurroundingTextService {
    }


-    private Pair<String, String> findSurroundingText(SectionText sectionText, String value,
-                                                     List<Rectangle> toFindPositions) {
+    private Pair<String, String> findSurroundingText(SectionText sectionText, String value, List<Rectangle> toFindPositions) {

-        Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), Set.of(value), "dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, false, EntityType.ENTITY);
+        Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), new SearchImplementation(value, false),
+                new FindEntityDetails("dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, EntityType.ENTITY));
        Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null);

        Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);
@@ -5,10 +5,13 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Commen
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
@@ -29,9 +32,44 @@ public class RedactionLogMergeService {

    private final SectionTextService sectionTextService;

+    private final RedactionStorageService redactionStorageService;

-    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
-                                              Set<Integer> excludedPages, List<Type> types, Colors colors) {
+    public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
+        log.debug("Requested preview for: {}", redactionRequest);
+
+        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
+
+        if (redactionLog == null) {
+            throw new NotFoundException("RedactionLog not present");
+        }
+
+        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
+        if (sectionGrid.getSections().isEmpty()) {
+
+            log.info("SectionGrid does not have headlines set. Computing headlines now!");
+            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
+
+            // enhance section grid with headline data
+            for (var sectionText : text.getSectionTexts()) {
+                sectionGrid.getSections()
+                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText.getSectionAreas()
+                                .stream()
+                                .map(SectionArea::getPage)
+                                .collect(Collectors.toSet()), sectionText.getSectionAreas()));
+            }
+            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
+        }
+
+        log.debug("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
+        var merged = mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
+
+        merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
+
+        return merged;
+    }
+
+    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
+                                               Set<Integer> excludedPages, List<Type> types, Colors colors) {

        var skippedImportedRedactions = new HashSet<>();

@@ -55,7 +93,7 @@ public class RedactionLogMergeService {
                    skippedImportedRedactions.add(entry.getId());
                }

-                entry.setComments(manualRedactions.getComments().get(entry.getId()));
+                entry.setComments(convert(manualRedactions.getComments().get(entry.getId())));

                if (excludedPages != null && !excludedPages.isEmpty()) {
                    entry.getPositions().forEach(pos -> {
@@ -297,7 +335,7 @@ public class RedactionLogMergeService {
            if (shouldCreateManualEntry(manualRedactionEntry)) {
                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types);
                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
-                redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
+                redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId())));
                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
                redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());

@@ -310,6 +348,11 @@ public class RedactionLogMergeService {
        return redactionLogEntries;
    }

+    private List<RedactionLogComment> convert(List<Comment> comments) {
+        return comments == null ? null :comments.stream().map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(),
+                c.getFileId(), c.getDate(), c.getSoftDeletedTime())).collect(Collectors.toList());
+    }
+

    private List<Rectangle> convertPositions(
            List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions) {
@@ -345,6 +388,7 @@ public class RedactionLogMergeService {
                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
                .legalBasis(manualRedactionEntry.getLegalBasis())
                .value(manualRedactionEntry.getValue())
+                .sourceId(manualRedactionEntry.getSourceId())
                .section(manualRedactionEntry.getSection())
                .type(manualRedactionEntry.getType())
                .redacted(true)
@@ -46,7 +46,7 @@ public class SectionGridCreatorService {
                            .getRectanglesPerPage()
                            .computeIfAbsent(page, (x) -> new ArrayList<>())
                            .add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
-                                    .getHeight(), i + 1, paragraph.getPageBlocks().size()));
+                                    .getHeight(), i + 1, paragraph.getPageBlocks().size(),null));

                } else if (textBlock instanceof Table) {

@@ -79,7 +79,7 @@ public class SectionTextBuilderService {
                        .get(0)
                        .getSequences()
                        .get(0)
-                        .getPage());
+                        .getPage(), null);
                sectionText.getSectionAreas().add(sectionArea);
                sectionText.getTextBlocks().addAll(cell.getTextBlocks());

@@ -141,7 +141,7 @@ public class SectionTextBuilderService {
                        .get(0)
                        .getSequences()
                        .get(0)
-                        .getPage());
+                        .getPage(), null);
                sectionText.getTextBlocks().addAll(cell.getTextBlocks());
                sectionText.getSectionAreas().add(sectionArea);

@@ -171,11 +171,11 @@ public class SectionTextBuilderService {
        SectionText sectionText = new SectionText();
        for (TextBlock paragraphTextBlock : paragraphTextBlocks) {
            SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock
-                    .getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage());
+                    .getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage(), null);
            sectionText.getSectionAreas().add(sectionArea);
        }

-        sectionText.setText(searchableText.toString());
+        sectionText.setText(searchableText.asString());
        sectionText.setHeadline(headline);
        sectionText.setSectionNumber(sectionNumber.intValue());
        sectionText.setTable(false);
@@ -31,7 +31,7 @@ public class SurroundingWordsService {
                if (dictionary != null && dictionary.isHint(entity.getType())) {
                    continue;
                }
-                findSurroundingWords(entity, searchableText.toString(), entity.getStart(), entity.getEnd());
+                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd());
            }
        } catch (Exception e) {
            log.warn("Could not get surrounding text!");
@@ -47,7 +47,7 @@ public class SurroundingWordsService {
        }

        try {
-            String searchableString = searchableText.toString();
+            String searchableString = searchableText.asString();

            if (cellstarts != null) {
                for (int i = 0; i < cellstarts.size(); i++) {
@@ -1,112 +1,57 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
 public class EntitySearchUtils {

-    public boolean sectionContainsAny(String sectionText, Set<DictionaryIncrementValue> values) {

-        String inputString = sectionText.toLowerCase(Locale.ROOT);
-
-        for (DictionaryIncrementValue value : values) {
-
-            String cleanValue = value.getValue().toLowerCase(Locale.ROOT).trim();
-
-            if (cleanValue.length() <= 2) {
-                continue;
-            }
-
-            int startIndex;
-            int stopIndex = 0;
-            do {
-                startIndex = inputString.indexOf(cleanValue, stopIndex);
-                stopIndex = startIndex + cleanValue.length();
-
-                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    return true;
-                }
-            } while (startIndex > -1);
-        }
-        return false;
+    public boolean sectionContainsAny(String sectionText, SearchImplementation searchImplementation) {
+        return searchImplementation.atLeastOneMatches(sectionText);
    }

+    public Set<Entity> findEntities(String inputString, SearchImplementation searchImplementation, DictionaryModel type, FindEntityDetails details) {

-    public Set<Entity> findEntities(String inputString, Set<String> values, DictionaryModel type, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionary,
-                                    Engine engine, boolean ignoreMinLength, boolean asRecommendation) {
+        Set<Entity> found = find(inputString, searchImplementation, details);

-       Set<Entity> found = find(inputString, values, type.getType(),  headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY);
-
-       if(asRecommendation){
-           Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationValues(), type.getType(),  headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_RECOMMENDATION);
-           removeFalsePositives(found, falseRecommendations);
-           found.addAll(falseRecommendations);
-       } else {
-           Set<Entity> falsePositives = find(inputString, type.getFalsePositiveValues(), type.getType(),  headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_POSITIVE);
-           removeFalsePositives(found, falsePositives);
-           found.addAll(falsePositives);
-       }
-
-       return found;
-    }
-
-
-    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionary,
-                                    Engine engine, boolean ignoreMinLength, EntityType entityType) {
-        Set<Entity> found = new HashSet<>();
-
-        for (String value : values) {
-
-            String cleanValue = value.trim();
-
-            if (!ignoreMinLength && cleanValue.length() <= 2) {
-                continue;
-            }
-
-            int startIndex;
-            int stopIndex = 0;
-            do {
-                startIndex = inputString.indexOf(cleanValue, stopIndex);
-                stopIndex = startIndex + cleanValue.length();
-
-                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, entityType));
-                }
-            } while (startIndex > -1);
+        if (details.getEntityType() == EntityType.RECOMMENDATION) {
+            Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
+            removeFalsePositives(found, falseRecommendations);
+            found.addAll(falseRecommendations);
+        } else {
+            Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
+            removeFalsePositives(found, falsePositives);
+            found.addAll(falsePositives);
        }
+
        return found;
    }

+    public Set<Entity> find(String inputString, SearchImplementation searchImplementation, FindEntityDetails findEntityDetails) {
+        Set<Entity> entities = new HashSet<>();

+        searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.getStartIndex(), match.getEndIndex()));

-    private boolean isSeparator(char c) {
+        return entities;
+    }

-        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
+    private void validateAndAddEntity(Set<Entity> entities, FindEntityDetails findEntityDetails, String inputString, int startIndex, int stopIndex) {
+
+        if ((startIndex == 0 || SeparatorUtils.isSeparator(inputString.charAt(startIndex - 1)))
+                && (stopIndex == inputString.length() || SeparatorUtils.isSeparator(inputString.charAt(stopIndex)))) {
+            entities.add(new Entity(inputString.substring(startIndex, stopIndex), findEntityDetails.getType(), startIndex, stopIndex,
+                    findEntityDetails.getHeadline(), findEntityDetails.getSectionNumber(), findEntityDetails.isDictionaryEntry(),
+                    findEntityDetails.isDossierDictionary(), findEntityDetails.getEngine(), findEntityDetails.getEntityType()));
+        }
    }


@@ -140,7 +85,6 @@ public class EntitySearchUtils {
    }


-
    public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {

        List<Entity> wordsToRemove = new ArrayList<>();
@@ -148,17 +92,15 @@ public class EntitySearchUtils {
            for (Entity inner : entities) {
                if (inner.getWord().length() < word.getWord()
                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
-                        wordsToRemove.add(inner);
+                    wordsToRemove.add(inner);
                }
            }
        }
-        entities.removeAll(wordsToRemove);
+        wordsToRemove.forEach(entities::remove);
        entities.removeAll(falsePositives);
    }


-
-
    public void removeEntitiesContainedInLarger(Set<Entity> entities) {

        List<Entity> wordsToRemove = new ArrayList<>();
@@ -168,7 +110,8 @@ public class EntitySearchUtils {
                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                        wordsToRemove.add(word);
-                    } else {
+                    } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
+                            inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
                        wordsToRemove.add(inner);
                    }
                }
@@ -216,17 +159,20 @@ public class EntitySearchUtils {

        if (entities.contains(found)) {
            Optional<Entity> existingOptional = entities.stream().filter(entity -> entity.equals(found)).findFirst();
-            if (!existingOptional.isPresent()) {
+            if (existingOptional.isEmpty()) {
                return;
            }
            var existing = existingOptional.get();

            if (existing.getType().equals(found.getType())) {
                existing.getEngines().addAll(found.getEngines());
-                if(existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
-                        || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)){
+                existing.setLegalBasis(found.getLegalBasis());
+                existing.setMatchedRule(found.getMatchedRule());
+                existing.setRedactionReason(found.getRedactionReason());
+                if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
+                        || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
                    existing.setEntityType(EntityType.ENTITY);
-                    if(found.isRedaction()){
+                    if (found.isRedaction()) {
                        existing.setRedaction(true);
                    }
                }
@@ -282,11 +228,14 @@ public class EntitySearchUtils {
        for (Entity toAdd : toBeAdded) {
            if (existing.contains(toAdd)) {
                Optional<Entity> existingOptional = existing.stream().filter(entity -> entity.equals(toAdd)).findFirst();
-                if (!existingOptional.isPresent()) {
+                if (existingOptional.isEmpty()) {
                    return;
                }
                var existingEntity = existingOptional.get();
                existingEntity.getEngines().addAll(toAdd.getEngines());
+                existingEntity.setLegalBasis(toAdd.getLegalBasis());
+                existingEntity.setMatchedRule(toAdd.getMatchedRule());
+                existingEntity.setRedactionReason(toAdd.getRedactionReason());
            } else {
                existing.add(toAdd);
            }
@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class FindEntityDetails {
+
+    private String type;
+    private String headline;
+    private int sectionNumber;
+    private boolean isDictionaryEntry;
+    private boolean isDossierDictionary;
+    private Engine engine;
+    private EntityType entityType;
+
+    public FindEntityDetails withEntityType(EntityType entityType) {
+        this.entityType = entityType;
+        return this;
+    }
+}
@@ -0,0 +1,92 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.ahocorasick.trie.Trie;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+@Data
+public class SearchImplementation {
+
+    private boolean ignoreCase;
+    private List<String> values;
+
+    private Pattern pattern;
+    private Trie trie;
+
+    public SearchImplementation(String value, boolean ignoreCase) {
+        this.values = List.of(value);
+        this.ignoreCase = ignoreCase;
+        this.createSearchImplementation();
+    }
+
+    public SearchImplementation(Collection<String> values, boolean ignoreCase) {
+        this.values = new ArrayList<>(values);
+        this.ignoreCase = ignoreCase;
+        this.createSearchImplementation();
+    }
+
+    private void createSearchImplementation() {
+        if (this.values.isEmpty()) {
+            return;
+        }
+
+        if (this.values.size() == 1) {
+            var text = this.values.iterator().next();
+            this.pattern = Pattern.compile(Pattern.quote(ignoreCase ? text.toLowerCase(Locale.ROOT) : text));
+        } else {
+            var builder = Trie.builder();
+            if (this.ignoreCase) {
+                builder.ignoreCase();
+            }
+
+            builder.addKeywords(this.values);
+
+            this.trie = builder.build();
+        }
+    }
+
+    public boolean atLeastOneMatches(String text) {
+        String textToCheck = text;
+        if (this.values.isEmpty()) {
+            return false;
+        }
+        if (this.pattern != null) {
+            if (ignoreCase) {
+                textToCheck = textToCheck.toLowerCase(Locale.ROOT);
+            }
+            return this.pattern.matcher(textToCheck).results().findAny().isPresent();
+        } else {
+            return this.trie.containsMatch(textToCheck);
+        }
+
+    }
+
+    public List<MatchPosition> getMatches(String text) {
+        String textToCheck = text;
+        if (this.values.isEmpty()) {
+            return new ArrayList<>();
+        }
+        if (this.pattern != null) {
+            if (ignoreCase) {
+                textToCheck = textToCheck.toLowerCase(Locale.ROOT);
+            }
+            return this.pattern.matcher(textToCheck).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList());
+        } else {
+            return this.trie.parseText(textToCheck).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList());
+        }
+    }
+
+    @Data
+    @AllArgsConstructor
+    public static class MatchPosition {
+        private int startIndex;
+        private int endIndex;
+    }
+}
@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.util.Set;
+import java.util.regex.Pattern;
+
+public class SeparatorUtils {
+
+    private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E'
+            , '\u2039', '\u203A');
+
+
+    public static boolean isSeparator(char c) {
+        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || quotes.contains(c);
+    }
+}
@@ -36,7 +36,7 @@ public class ImageService {
        ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class);

        Map<Integer, List<PdfImage>> images = new HashMap<>();
-        imageServiceResponse.getData().stream().forEach(imageMetadata -> {
+        imageServiceResponse.getData().forEach(imageMetadata -> {
            var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)) : ImageType.OTHER;
            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), classification, imageMetadata.getPosition().getPageNumber()));
@@ -1,5 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;

+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
+import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.pdfbox.io.MemoryUsageSetting;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.springframework.stereotype.Service;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -13,38 +35,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.SystemUtils;
-import org.apache.pdfbox.io.MemoryUsageSetting;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
-import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
-import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
-import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
@Slf4j
@Service
@RequiredArgsConstructor
 public class PdfSegmentationService {

-    private final static int MAX_PAGES_BEFORE_GC = 250;
-
    private final RulingCleaningService rulingCleaningService;
    private final TableExtractionService tableExtractionService;
    private final BlockificationService blockificationService;
@@ -60,11 +55,10 @@ public class PdfSegmentationService {
            //create tempFile
            File tempFile;

-            if(SystemUtils.IS_OS_UNIX) {
+            if (SystemUtils.IS_OS_UNIX) {
                FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
                tempFile = Files.createTempFile("document", ".pdf", attr).toFile();
-            }
-            else {
+            } else {
                tempFile = Files.createTempFile("document", ".pdf").toFile();
                tempFile.setReadable(true, true);
                tempFile.setWritable(true, true);
@@ -78,15 +72,12 @@ public class PdfSegmentationService {
                Document document = new Document();
                List<Page> pages = new ArrayList<>();

-                pdDocument = reinitializePDDocument(tempFile, null);
+                pdDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupMixed(1024 * 1024 * 64));
+                pdDocument.setAllSecurityToBeRemoved(true);
                long pageCount = pdDocument.getNumberOfPages();

                for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {

-                    if (pageNumber % MAX_PAGES_BEFORE_GC == 0) {
-                        pdDocument = reinitializePDDocument(tempFile, pdDocument);
-                    }
-
                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
                    stripper.setPageNumber(pageNumber);
@@ -107,6 +98,7 @@ public class PdfSegmentationService {
                    Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
                            .getVertical());

+
                    PDRectangle cropbox = pdPage.getCropBox();
                    float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
                    page.setCropBoxArea(cropboxArea);
@@ -134,11 +126,10 @@ public class PdfSegmentationService {
                sectionsBuilderService.buildSections(document);
                sectionsBuilderService.addImagesToSections(document);

-                pdDocument = reinitializePDDocument(tempFile, pdDocument);

                IOUtils.close(pdDocument);

-                if(!tempFile.delete()){
+                if (!tempFile.delete()) {
                    log.warn("Could not delete tmp file");
                }

@@ -152,23 +143,6 @@ public class PdfSegmentationService {
    }


-    private PDDocument reinitializePDDocument(File tempFile, PDDocument pdDocument) throws IOException {
-
-        if (pdDocument != null) {
-            pdDocument.close();
-        }
-        System.runFinalization();
-        System.gc();
-
-        MemoryStats.printMemoryStats();
-
-        var newPDDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupTempFileOnly());
-        newPDDocument.setAllSecurityToBeRemoved(true);
-
-        return newPDDocument;
-    }
-
-
    private void increaseDocumentStatistics(Page page, Document document) {

        if (!page.isLandscape()) {
@@ -1,16 +1,31 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;

-import com.iqser.red.service.redaction.v1.server.classification.model.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.stereotype.Service;

-import java.util.*;
-import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;

+@Slf4j
@Service
 public class SectionsBuilderService {

@@ -53,8 +68,7 @@ public class SectionsBuilderService {
                    continue;
                }

-                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification()
-                        .startsWith("H ") || !document.isHeadlines()) {
+                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                    chunkBlock.setHeadline(lastHeadline);
                    if (document.isHeadlines()) {
@@ -100,17 +114,19 @@ public class SectionsBuilderService {

    public void addImagesToSections(Document document) {

-        Map<Integer, SortedSet<Paragraph>> paragraphMap = new HashMap<>();
+        Map<Integer, List<Paragraph>> paragraphMap = new HashMap<>();
        for (Paragraph paragraph : document.getParagraphs()) {
            for (AbstractTextContainer container : paragraph.getPageBlocks()) {
-                paragraphMap.computeIfAbsent(container.getPage(), x -> new TreeSet<>()).add(paragraph);
+
+                paragraphMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()).add(paragraph);
+
            }
        }

        if (paragraphMap.isEmpty()) {
            Paragraph paragraph = new Paragraph();
            document.getParagraphs().add(paragraph);
-            paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
+            paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
        }

        // first page is always a paragraph, else we can't process pages 1..N,
@@ -118,12 +134,12 @@ public class SectionsBuilderService {
        if (paragraphMap.get(1) == null) {
            Paragraph paragraph = new Paragraph();
            document.getParagraphs().add(paragraph);
-            paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
+            paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
        }

        for (Page page : document.getPages()) {
            for (PdfImage image : page.getImages()) {
-                SortedSet<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
+                List<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
                if (paragraphsOnPage == null) {
                    int i = page.getPageNumber();
                    while (paragraphsOnPage == null) {
@@ -131,27 +147,63 @@ public class SectionsBuilderService {
                        i--;
                    }
                }
-
-                Float perviousEnd = 0f;
                for (Paragraph paragraph : paragraphsOnPage) {
-                    Float currentEnd = 0f;
+                    Float xMin = null;
+                    Float yMin = null;
+                    Float xMax = null;
+                    Float yMax = null;
+
                    for (AbstractTextContainer abs : paragraph.getPageBlocks()) {
                        if (abs.getPage() != page.getPageNumber()) {
                            continue;
                        }
-                        if (abs.getMaxY() > currentEnd) {
-                            currentEnd = abs.getMaxY();
+
+                        if (abs.getMinX() < abs.getMaxX()) {
+                            if (xMin == null || abs.getMinX() < xMin) {
+                                xMin = abs.getMinX();
+                            }
+                            if (xMax == null || abs.getMaxX() > xMax) {
+                                xMax = abs.getMaxX();
+                            }
+                        } else {
+                            if (xMin == null || abs.getMaxX() < xMin) {
+                                xMin = abs.getMaxX();
+                            }
+                            if (xMax == null || abs.getMinX() > xMax) {
+                                xMax = abs.getMinX();
+                            }
                        }
+
+                        if (abs.getMinY() < abs.getMaxY()) {
+                            if (yMin == null || abs.getMinY() < yMin) {
+                                yMin = abs.getMinY();
+                            }
+                            if (yMax == null || abs.getMaxY() > yMax) {
+                                yMax = abs.getMaxY();
+                            }
+                        } else {
+                            if (yMin == null || abs.getMaxY() < yMin) {
+                                yMin = abs.getMaxY();
+                            }
+                            if (yMax == null || abs.getMinY() > yMax) {
+                                yMax = abs.getMinY();
+                            }
+                        }
+
                    }

-                    if (image.getPosition().getY() >= perviousEnd && image.getPosition().getY() <= currentEnd) {
+                    log.debug("Image position x: {}, y: {}", image.getPosition().getX(), image.getPosition().getY());
+                    log.debug("Paragraph position xMin: {}, xMax: {}, yMin: {}, yMax: {}", xMin, xMax, yMin, yMax);
+
+                    if (xMin != null && xMax != null && yMin != null && yMax != null && image.getPosition().getX() >= xMin && image.getPosition()
+                            .getX() <= xMax && image.getPosition().getY() >= yMin && image.getPosition().getY() <= yMax) {
                        paragraph.getImages().add(image);
                        image.setAppendedToParagraph(true);
                    }
-                    perviousEnd = currentEnd;
                }
                if (!image.isAppendedToParagraph()) {
-                    paragraphsOnPage.first().getImages().add(image);
+                    log.debug("Image uses first paragraph");
+                    paragraphsOnPage.get(0).getImages().add(image);
                    image.setAppendedToParagraph(true);
                }
            }
@@ -166,9 +218,7 @@ public class SectionsBuilderService {
            List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
            List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
            // Allow merging of tables if header row is separated from first logical non-header row
-            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows()
-                    .get(0)
-                    .size() == tableNonHeaderRow.size()) {
+            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows().get(0).size() == tableNonHeaderRow.size()) {
                previousTableNonHeaderRow = previousTable.getRows().get(0).stream().map(cell -> {
                    Cell fakeCell = new Cell(cell.getPoints()[0], cell.getPoints()[2]);
                    fakeCell.setHeaderCells(Collections.singletonList(cell));
@@ -178,8 +228,7 @@ public class SectionsBuilderService {
            if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
                for (int i = currentTable.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
                    List<Cell> row = currentTable.getRows().get(i);
-                    if (row.size() == tableNonHeaderRow.size() && row.stream()
-                            .allMatch(cell -> cell.getHeaderCells().isEmpty())) {
+                    if (row.size() == tableNonHeaderRow.size() && row.stream().allMatch(cell -> cell.getHeaderCells().isEmpty())) {
                        for (int j = 0; j < row.size(); j++) {
                            row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
                        }
@@ -229,24 +278,20 @@ public class SectionsBuilderService {
            TextBlock wordBlock = (TextBlock) container;

            if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
-                        .getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
                textBlock.setPage(wordBlock.getPage());
            } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
-                        .getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
                textBlock.setPage(wordBlock.getPage());
                alreadyAdded = false;
            } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
                textBlock.setPage(pageBefore);
                paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
-                        .getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
                textBlock.setPage(wordBlock.getPage());
            } else {
                TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
-                        .getHeight());
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
            }
            pageBefore = wordBlock.getPage();
            splitByTable = false;
@@ -268,11 +313,7 @@ public class SectionsBuilderService {

    private boolean hasInvalidHeaderInformation(Table table) {

-        return table.getRows()
-                .stream()
-                .flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
-                .findAny()
-                .isEmpty();
+        return table.getRows().stream().flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells()))).findAny().isEmpty();

    }

@@ -1,11 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.storage;

-import java.io.IOException;
-import java.io.InputStream;
-
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.stereotype.Service;
-
+import com.dslplatform.json.DslJson;
+import com.dslplatform.json.runtime.Settings;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
@@ -16,19 +12,26 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.stereotype.Service;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;

@Slf4j
@Service
@RequiredArgsConstructor
 public class RedactionStorageService {

-    private final ObjectMapper objectMapper;
    private final StorageService storageService;
+    private final ObjectMapper objectMapper;
+
+    private final DslJson<Object> dslJson = new DslJson<>(Settings.basicSetup());


    @SneakyThrows
@@ -40,8 +43,23 @@ public class RedactionStorageService {

    @SneakyThrows
    public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
+        var baos = new ByteArrayOutputStream();
+        try {
+            dslJson.serialize(any, baos);
+        } catch (com.dslplatform.json.SerializationException e){
+            // Fails on file 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
+            var bytes = objectMapper.writeValueAsBytes(any);
+            storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), bytes);
+            dslJson.newWriter();
+            return;
+        }
+        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), baos.toByteArray());
+    }

-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any));
+    @SneakyThrows
+    public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) {
+
+        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
    }


@@ -56,7 +74,7 @@ public class RedactionStorageService {
        }

        try {
-            return objectMapper.readValue(inputStreamResource.getInputStream(), ImportedRedactions.class);
+            return dslJson.deserialize(ImportedRedactions.class, inputStreamResource.getInputStream());
        } catch (IOException e) {
            throw new RuntimeException("Could not imported redactions", e);
        }
@@ -74,7 +92,7 @@ public class RedactionStorageService {
        }

        try {
-            return objectMapper.readValue(inputStreamResource.getInputStream(), RedactionLog.class);
+            return dslJson.deserialize(RedactionLog.class, inputStreamResource.getInputStream());
        } catch (IOException e) {
            throw new RuntimeException("Could not convert RedactionLog", e);
        }
@@ -92,7 +110,7 @@ public class RedactionStorageService {
        }

        try {
-            return objectMapper.readValue(inputStreamResource.getInputStream(), Text.class);
+            return dslJson.deserialize(Text.class, inputStreamResource.getInputStream());
        } catch (IOException e) {
            throw new RuntimeException("Could not convert Text", e);
        }
@@ -109,7 +127,7 @@ public class RedactionStorageService {
        }

        try {
-            return objectMapper.readValue(inputStreamResource.getInputStream(), NerEntities.class);
+            return dslJson.deserialize(NerEntities.class, inputStreamResource.getInputStream());
        } catch (IOException e) {
            throw new RuntimeException("Could not convert NER Entities", e);
        }
@@ -120,7 +138,7 @@ public class RedactionStorageService {

        try {
            var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
-            return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class);
+            return dslJson.deserialize(SectionGrid.class, sectionGrid.getInputStream());
        } catch (StorageObjectDoesNotExist e) {
            throw new NotFoundException("Section Grid is not available.");
        } catch (IOException e) {
@@ -1,9 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -33,11 +33,13 @@ public abstract class AbstractTextContainer {
    }

    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getHeight() {
        return maxY - minY;
    }
-    
+
    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getWidth() {
        return maxX - minX;
    }
@@ -0,0 +1,115 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;
+
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
+public class DictionaryServiceTest {
+
+    @MockBean
+    protected KieContainer kieContainer;
+
+    @MockBean
+    protected DictionaryClient dictionaryClient;
+
+    @Autowired
+    protected DictionaryService dictionaryService;
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+    @Test
+    public void testDictionaryServiceConsistency() {
+
+        when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
+        when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId", "#cccccc",
+                "#cccccc", "#cccccc", "#cccccc", "#cccccc", "#cccccc",
+                "#cccccc", "#cccccc", "#cccccc", "#cccccc"));
+
+        var type1 = new Type();
+        type1.setType("type1");
+        type1.setId("type1");
+        type1.setVersion(1L);
+        type1.setHexColor("#cccccc");
+        type1.setHasDictionary(true);
+        type1.setEntries(Stream.of("a", "b", "c").map(t -> new DictionaryEntry(1, t, 1L, false, "type1")).collect(Collectors.toList()));
+
+
+        var type2 = new Type();
+        type2.setType("type2");
+        type2.setId("type2");
+        type2.setVersion(1L);
+        type2.setHexColor("#cccccc");
+        type2.setHasDictionary(true);
+        type2.setEntries(Stream.of("d", "e", "f").map(t -> new DictionaryEntry(1, t, 1L, false, "type2")).collect(Collectors.toList()));
+
+
+        var type1Updated = new Type();
+        type1Updated.setType("type1");
+        type1Updated.setId("type1");
+        type1Updated.setVersion(2L);
+        type1Updated.setHexColor("#cccccc");
+        type1Updated.setHasDictionary(true);
+        type1Updated.setEntries(Stream.of("z", "q", "x").map(t -> new DictionaryEntry(1, t, 2L, false, "type1")).collect(Collectors.toList()));
+
+
+        when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1, type2));
+        when(dictionaryClient.getDictionaryForType("type1", null)).thenReturn(type1);
+        when(dictionaryClient.getDictionaryForType("type2", null)).thenReturn(type2);
+        when(dictionaryClient.getDictionaryForType("type1", 0L)).thenReturn(type1);
+        when(dictionaryClient.getDictionaryForType("type2", 0L)).thenReturn(type2);
+
+        dictionaryService.updateDictionary("dossierTemplateId", null);
+        var increments = dictionaryService.getDictionaryIncrements("dossierTemplateId",
+                DictionaryVersion.builder().dossierTemplateVersion(0L).build(), null);
+
+
+        when(dictionaryClient.getVersion(any())).thenReturn(1L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1Updated));
+        when(dictionaryClient.getDictionaryForType("type1", 1L)).thenReturn(type1Updated);
+
+        dictionaryService.updateDictionary("dossierTemplateId", null);
+        var updatedIncrements = dictionaryService.getDictionaryIncrements("dossierTemplateId",
+                DictionaryVersion.builder().dossierTemplateVersion(0L).build(), null);
+
+    }
+}
@@ -10,15 +10,15 @@ import org.springframework.core.io.InputStreamResource;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.InputStream;
+import java.util.*;
+import java.util.stream.Collectors;

-public class FileSystemBackedStorageService extends S3StorageService {
+public class FileSystemBackedStorageService implements StorageService{

    private final Map<String, File> dataMap = new HashMap<>();

    public FileSystemBackedStorageService() {
-        super(null, null);
    }

    @SneakyThrows
@@ -33,16 +33,51 @@ public class FileSystemBackedStorageService extends S3StorageService {

    }

+    @Override
+    public void deleteObject(String objectId) {
+        dataMap.remove(objectId);
+    }
+
+    @Override
+    public boolean objectExists(String objectId) {
+        return dataMap.containsKey(objectId);
+    }
+
+    @Override
+    public void init() {
+
+    }
+
+    public List<String> listPaths(){
+        return new ArrayList<>(dataMap.keySet());
+    }
+
+
+    public List<String> listFilePaths(){
+        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
+    }
+
+
    @SneakyThrows
    @Override
    public void storeObject(String objectId, byte[] data) {
-        File tempFile = File.createTempFile("test", ".tmp");
+        File tempFile = File.createTempFile("storage", objectId.replace("/","-"));

        IOUtils.write(data, new FileOutputStream(tempFile));

        dataMap.put(objectId, tempFile);
    }

+    @Override
+    @SneakyThrows
+    public void storeObject(String objectId, InputStream stream) {
+        File tempFile = File.createTempFile("test", ".tmp");
+
+        IOUtils.copy(stream, new FileOutputStream(tempFile));
+
+        dataMap.put(objectId, tempFile);
+    }
+
    public void clearStorage() {
        this.dataMap.forEach((k, v) -> {
            v.delete();
@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server;

 import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
@@ -13,12 +14,14 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
-import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
@@ -56,6 +59,7 @@ import java.io.*;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
 import java.util.*;
 import java.util.stream.Collectors;

@@ -91,6 +95,9 @@ public class RedactionIntegrationTest {
    @Autowired
    private RedactionController redactionController;

+    @Autowired
+    private AnnotationService annotationService;
+
    @Autowired
    private AnalyzeService analyzeService;

@@ -237,7 +244,6 @@ public class RedactionIntegrationTest {
    public void test270Rotated() {

        AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf");
-        MemoryStats.printMemoryStats();
        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        AnalyzeResult result = analyzeService.analyze(request);
        assertThat(result).isNotNull();
@@ -249,7 +255,6 @@ public class RedactionIntegrationTest {
    public void testLargeScannedFileOOM() {

        AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
-        MemoryStats.printMemoryStats();
        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        AnalyzeResult result = analyzeService.analyze(request);
        assertThat(result).isNotNull();
@@ -279,7 +284,7 @@ public class RedactionIntegrationTest {
        dictionary.get(AUTHOR).add("Drinking water");
        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -516,7 +521,7 @@ public class RedactionIntegrationTest {
        end = System.currentTimeMillis();
        System.out.println("reanalysis analysis duration: " + (end - start));

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -580,7 +585,7 @@ public class RedactionIntegrationTest {

        assertThat(changes.size()).isEqualTo(2);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -687,7 +692,7 @@ public class RedactionIntegrationTest {
        end = System.currentTimeMillis();
        System.out.println("reanalysis analysis duration: " + (end - start));

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -712,6 +717,72 @@ public class RedactionIntegrationTest {
    }


+    @Test
+    public void testRemovePublishedInformations() throws IOException {
+
+        long start = System.currentTimeMillis();
+
+        ClassPathResource colorsResource = new ClassPathResource("colors/colors.json");
+        var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
+
+        ClassPathResource typeResource = new ClassPathResource("colors/types.json");
+        TypeReference<List<Type>> typeRefForTypes = new TypeReference<>(){};
+        List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
+
+
+        AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        ManualRedactions manualRedactions = new ManualRedactions();
+        manualRedactions.getIdsToRemove().add(IdRemoval.builder()
+                .annotationId("308dab9015bfafd911568cffe0a7f7de")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.of(2022,05,23,8,30,07,475479,  ZoneOffset.UTC))
+                        .processedDate(OffsetDateTime.of(2022,05,23,8,30,07,483651,  ZoneOffset.UTC))
+                .build());
+
+        manualRedactions.getForceRedactions().add(ManualForceRedaction.builder()
+                .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.of(2022,05,23,9,30,15,4653, ZoneOffset.UTC))
+                .processedDate(OffsetDateTime.of(2022,05,23,9,30,15,794,  ZoneOffset.UTC))
+                .build());
+
+        manualRedactions.getIdsToRemove().add(IdRemoval.builder()
+                .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.of(2022,05,23,8,30,23,961721,  ZoneOffset.UTC))
+                .processedDate(OffsetDateTime.of(2022,05,23,8,30,23,96528,  ZoneOffset.UTC))
+                .build());
+
+
+
+        request.setManualRedactions(manualRedactions);
+
+
+        AnalyzeResult result = analyzeService.analyze(request);
+
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                        .manualRedactions(manualRedactions)
+                        .colors(colors)
+                        .types(types)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
    @Test
    public void testTableRedaction() throws IOException {

@@ -722,7 +793,7 @@ public class RedactionIntegrationTest {
        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        AnalyzeResult result = analyzeService.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -812,7 +883,7 @@ public class RedactionIntegrationTest {

        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -946,7 +1017,7 @@ public class RedactionIntegrationTest {
        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        AnalyzeResult result = analyzeService.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -1078,7 +1149,7 @@ public class RedactionIntegrationTest {

        analyzeService.reanalyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
@@ -1102,7 +1173,7 @@ public class RedactionIntegrationTest {
        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        AnalyzeResult result = analyzeService.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -1179,7 +1250,7 @@ public class RedactionIntegrationTest {
        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        AnalyzeResult result = analyzeService.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
                .fileId(TEST_FILE_ID)
@@ -1526,7 +1597,7 @@ public class RedactionIntegrationTest {

        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -1555,7 +1626,7 @@ public class RedactionIntegrationTest {
        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        analyzeService.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -1,6 +1,10 @@
-package com.iqser.red.service.redaction.v1.model;
+package com.iqser.red.service.redaction.v1.server.annotate;
+
+import java.util.List;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -17,4 +21,6 @@ public class AnnotateRequest {
    private String dossierTemplateId;
    private String fileId;
    private ManualRedactions manualRedactions;
+    private Colors colors;
+    private List<Type> types;
 }
@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.model;
+package com.iqser.red.service.redaction.v1.server.annotate;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -1,9 +1,15 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
+package com.iqser.red.service.redaction.v1.server.annotate;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import lombok.RequiredArgsConstructor;
+import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
@@ -17,6 +23,7 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;

 import java.awt.*;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.*;
@@ -26,8 +33,41 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
 public class AnnotationService {

+    private final RedactionStorageService redactionStorageService;
+    private final DictionaryService dictionaryService;

-    public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
+    private final RedactionLogMergeService redactionLogMergeService;
+
+    public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
+
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
+        var mergedRedactionLog = redactionLogMergeService.provideRedactionLog(RedactionRequest.builder()
+                .fileId(annotateRequest.getFileId())
+                .manualRedactions(annotateRequest.getManualRedactions())
+                .dossierId(annotateRequest.getDossierId())
+                .dossierTemplateId(annotateRequest.getDossierTemplateId())
+                        .colors(annotateRequest.getColors())
+                        .types(annotateRequest.getTypes())
+                .build());
+        var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
+
+        try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
+            pdDocument.setAllSecurityToBeRemoved(true);
+
+            dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
+            annotate(pdDocument, mergedRedactionLog, sectionsGrid);
+
+            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
+                pdDocument.save(byteArrayOutputStream);
+                return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
+            }
+
+        } catch (Exception e) {
+            throw new RedactionException(e);
+        }
+    }
+
+    private void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {

        Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);

@@ -86,7 +126,7 @@ public class AnnotationService {
        annotations.add(annotation);

        if (redactionLogEntry.getComments() != null) {
-            for (Comment comment : redactionLogEntry.getComments()) {
+            for (RedactionLogComment comment : redactionLogEntry.getComments()) {
                PDAnnotationText txtAnnot = new PDAnnotationText();
                txtAnnot.setAnnotationName(String.valueOf(comment.getId()));
                txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation
@@ -0,0 +1,87 @@
+package com.iqser.red.service.redaction.v1.server.realdata;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.MessageType;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+import org.springframework.core.io.ClassPathResource;
+
+import java.time.OffsetDateTime;
+import java.util.List;
+import java.util.Set;
+
+@Slf4j
+public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest {
+
+    public static final String FILE_NAME = "test-file";
+
+    @Test
+    @SneakyThrows
+    public void testFile() {
+
+        ObjectMapper om = new ObjectMapper();
+        om.registerModule(new JavaTimeModule());
+
+        var file = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".pdf").getInputStream();
+        redactionStorageService.storeObject("dossierId", "fileId", FileType.ORIGIN, file);
+
+
+        try {
+            var nerData = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".ner.json").getInputStream();
+            redactionStorageService.storeObject("dossierId", "fileId", FileType.NER_ENTITIES, nerData);
+        } catch (Exception e) {
+            log.warn("No NER File Provided");
+            redactionServiceSettings.setNerServiceEnabled(false);
+        }
+
+
+        AnalyzeRequest ar = AnalyzeRequest.builder()
+                .fileId("fileId")
+                .dossierId("dossierId")
+                .analysisNumber(1)
+                .dossierTemplateId("dossierTemplateId")
+                .lastProcessed(OffsetDateTime.now())
+                .excludedPages(Set.of())
+                .fileAttributes(List.of())
+                .build();
+
+
+        try {
+            var text = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".text.json").getInputStream();
+            var sectionText = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".section-grid.json").getInputStream();
+            redactionStorageService.storeObject("dossierId", "fileId", FileType.TEXT, text);
+            redactionStorageService.storeObject("dossierId", "fileId", FileType.SECTION_GRID, sectionText);
+        } catch (Exception e) {
+            log.info("No text file provided, Performing Structure analysis");
+
+            ar.setMessageType(MessageType.STRUCTURE_ANALYSE);
+            String in = om.writeValueAsString(ar);
+            redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+        }
+
+
+        try {
+            var redactionLog = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".redaction-log.json").getInputStream();
+        } catch (Exception e) {
+            log.info("No redaction log provided, Performing full analysis");
+
+            ar.setMessageType(MessageType.ANALYSE);
+            String in = om.writeValueAsString(ar);
+            redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+        }
+
+
+        simulateIncrement(List.of("Desiree"), "PII", 3L);
+        ar.setMessageType(MessageType.REANALYSE);
+        String in = om.writeValueAsString(ar);
+        redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+
+
+        // var log = redactionStorageService.getRedactionLog("dossierId","fileId");
+    }
+
+}
@@ -0,0 +1,196 @@
+package com.iqser.red.service.redaction.v1.server.realdata;
+
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.*;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.SneakyThrows;
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.ResourcePatternResolver;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
+public class LiveDataIntegrationTest {
+
+    protected static String BASE_DIR = "performance/";
+
+    protected static String EFSA_SANITISATION_GFL_V1 = "dictionaries/EFSA_sanitisation_GFL_v1/";
+
+    @MockBean
+    protected DictionaryClient dictionaryClient;
+
+    @MockBean
+    protected EntityRecognitionClient entityRecognitionClient;
+
+    @MockBean
+    private FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
+
+    @MockBean
+    private LegalBasisClient legalBasisClient;
+
+    @Autowired
+    private ResourcePatternResolver resourcePatternResolver;
+
+    @MockBean
+    private RulesClient rulesClient;
+
+    @Autowired
+    protected DictionaryService dictionaryService;
+
+    @Autowired
+    protected RedactionStorageService redactionStorageService;
+
+    @Autowired
+    protected RedactionMessageReceiver redactionMessageReceiver;
+
+    @Autowired
+    protected FileSystemBackedStorageService fileSystemBackedStorageService;
+
+    @Autowired
+    protected RedactionServiceSettings redactionServiceSettings;
+
+    private List<Type> types;
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+
+    @SneakyThrows
+    @Before
+    public void prepareTest() {
+
+        when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
+        when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);
+
+        var rules = IOUtils.toString(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "rules.drl").getInputStream());
+        when(rulesClient.getRules(any())).thenReturn(JSONPrimitive.of(rules));
+
+        ObjectMapper objectMapper = new ObjectMapper();
+
+        var jsonNode = objectMapper.readTree(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "types.json").getInputStream());
+        types = objectMapper.treeToValue(jsonNode.get("types"), objectMapper.getTypeFactory().constructType(
+                new TypeReference<List<Type>>() {
+                }));
+
+        types.forEach(t -> {
+            t.setId(t.getType());
+            t.setVersion(0L);
+        });
+
+        when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(types);
+        when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>());
+
+
+        when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(), Colors.class));
+
+        when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
+            String typeName = answer.getArgument(0);
+
+            var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
+            if(found.isPresent()) {
+                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
+                type.setEntries(getEntries(typeName, type.getTypeId()));
+
+                return type;
+            }else{
+                return null;
+            }
+        });
+
+        dictionaryService.updateDictionary("dossierTemplateId","dossierId");
+    }
+
+    public void simulateIncrement(List<String> values, String deltaTypeName, long version) {
+        when(dictionaryClient.getVersion(anyString())).thenReturn(version);
+        when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(version);
+
+        when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
+            String typeName = answer.getArgument(0);
+
+            var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
+            if(found.isPresent()) {
+
+                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
+
+                if (deltaTypeName.equalsIgnoreCase(typeName)) {
+                    type.setEntries(values.stream().map(l -> new DictionaryEntry(0, l, version, false, type.getTypeId())).collect(Collectors.toList()));
+                } else {
+                    type.setEntries(new ArrayList<>());
+                }
+
+                return type;
+            }else{
+                return null;
+            }
+        });
+    }
+
+
+    @Test
+    public void testUpdateDictionary() {
+        dictionaryService.updateDictionary("dossierTemplateId", "dossierId");
+
+        var dict = dictionaryService.getDeepCopyDictionary("dossierTemplateId", "dossierId");
+        assertThat(dict.getLocalAccessMap().size()).isEqualTo(12);
+    }
+
+    @SneakyThrows
+    private List<DictionaryEntry> getEntries(String typeName, String typeId) {
+        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
+        for (var resource : dictionaryResources) {
+            if (resource.getFilename().contains(typeName)) {
+
+                List<String> lines = IOUtils.readLines(resource.getInputStream());
+                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
+
+            }
+        }
+        return new ArrayList<>();
+    }
+}
@@ -0,0 +1,85 @@
+package com.iqser.red.service.redaction.v1.server.stringmatching;
+
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.SneakyThrows;
+import org.ahocorasick.trie.Trie;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+@RunWith(SpringRunner.class)
+public class StringMatchingPerformanceTest {
+
+    @Test
+    @SneakyThrows
+    public void testStringPerformance() {
+
+        String text = IOUtils.toString(new ClassPathResource("stringmatching/hamlet.txt").getInputStream()).toLowerCase();
+        Set<String> dictionary = IOUtils.readLines(new ClassPathResource("stringmatching/names.txt").getInputStream())
+                .stream()
+                .map(String::toLowerCase).collect(Collectors.toSet());
+
+        System.out.println("Loaded text has a length of " + text.length() + " symbols");
+        System.out.println("Dictionary has " + dictionary.size() + " entries");
+
+        var patterns = dictionary.stream()
+                .map(p -> Pattern.compile(Pattern.quote(p))).collect(Collectors.toList());
+        var trie = Trie.builder().ignoreCase().addKeywords(dictionary).build();
+
+
+        // 1. Naive approach
+        long t1 = System.currentTimeMillis();
+        var naiveIndexes = new HashSet<Index>();
+        for (var entry : dictionary) {
+            var startIndex = 0;
+            do {
+                startIndex = text.indexOf(entry, startIndex + 1);
+                if (startIndex != -1) {
+                    naiveIndexes.add(new Index(startIndex, startIndex + entry.length()));
+                }
+            } while (startIndex != -1);
+        }
+        long t2 = System.currentTimeMillis();
+        System.out.println("Naive approach found " + naiveIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // 2. Boyer Moore
+        t1 = System.currentTimeMillis();
+        var boyerMooreIndexes = new HashSet<Index>();
+        for (var pattern : patterns) {
+            boyerMooreIndexes.addAll(pattern.matcher(text).results().map(r -> new Index(r.start(), r.end())).collect(Collectors.toList()));
+        }
+        t2 = System.currentTimeMillis();
+        System.out.println("Boyer Moore found " + boyerMooreIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // 3. Aho Corasick
+        t1 = System.currentTimeMillis();
+        var result = trie.parseText(text);
+        var ahoCorasickIndexes = result.stream().map(r -> new Index(r.getStart(), r.getEnd() + 1)).collect(Collectors.toSet());
+        t2 = System.currentTimeMillis();
+        System.out.println("Aho Corasick found " + ahoCorasickIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // Assert that all algorithms are equal
+        assertThat(naiveIndexes).isEqualTo(boyerMooreIndexes).isEqualTo(ahoCorasickIndexes);
+
+    }
+
+    @AllArgsConstructor
+    @EqualsAndHashCode(of = {"start", "end"})
+    public static class Index {
+        int start;
+        int end;
+    }
+}
@@ -0,0 +1,13 @@
+{
+  "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+  "defaultColor": "#9398a0",
+  "requestAdd": "#04b093",
+  "requestRemove": "#04b093",
+  "notRedacted": "#c498fa",
+  "analysisColor": "#dd4d50",
+  "updatedColor": "#fdbd00",
+  "dictionaryRequestColor": "#5b97db",
+  "manualRedactionColor": "#9398a0",
+  "previewColor": "#9398a0",
+  "ignoredHintColor": "#e7d4ff"
+}
@@ -0,0 +1,223 @@
+[
+  {
+    "id": "CBI_address:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "CBI_address",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 140,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "All site names and addresses, and location (e.g. Syngenta, Monthey, GPS Co-ordinates, Mr Smith of … providing the…). Except addresses in published literature and the applicant address.",
+    "addToDictionaryAction": true,
+    "label": "CBI Address",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "CBI_author:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "CBI_author",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 130,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "All authors named in the study documentation. Except names in published literature.",
+    "addToDictionaryAction": true,
+    "label": "CBI Author",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "PII:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "PII",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 150,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Not authors but listed in the document: Names, signatures, telephone, email etc.; e.g. Reg Manager, QA Manager",
+    "addToDictionaryAction": true,
+    "label": "PII",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "formula:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "formula",
+    "hexColor": "#036ffc",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1002,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure formula colors.",
+    "addToDictionaryAction": false,
+    "label": "Formula",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "isHint_only:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "isHint_only",
+    "hexColor": "#fa98f7",
+    "recommendationHexColor": "#8df06c",
+    "rank": 50,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Entries of this dictionary will be highlighted only",
+    "addToDictionaryAction": false,
+    "label": "isHint Only",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "image:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "image",
+    "hexColor": "#bdd6ff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 999,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure image colors.",
+    "addToDictionaryAction": false,
+    "label": "Image",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "logo:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "logo",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1001,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure logo colors.",
+    "addToDictionaryAction": false,
+    "label": "Logo",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "must_redact:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "must_redact",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 100,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Entries of this dictionary get redacted wherever found.",
+    "addToDictionaryAction": false,
+    "label": "Must Redact",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "ocr:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "ocr",
+    "hexColor": "#bdd6ff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1000,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure ocr colors.",
+    "addToDictionaryAction": false,
+    "label": "Ocr",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "signature:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "signature",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1003,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure signature colors.",
+    "addToDictionaryAction": false,
+    "label": "Signature",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "imported_redaction:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "imported_redaction",
+    "hexColor": "#f0f0c0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 9999,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Redaction Annotations that were imported from documents",
+    "addToDictionaryAction": false,
+    "label": "Imported Redaction",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": true
+  },
+  {
+    "id": "published_information:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "published_information",
+    "hexColor": "#85ebff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 70,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Manual managed list of public journals and papers that need no redaction",
+    "addToDictionaryAction": true,
+    "label": "Published Information",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "dossier_redaction:31039447-9040-4376-9ca7-614e56b284b9:5dfb2724-74a4-4a1a-a1eb-165e7943ffcd",
+    "type": "dossier_redaction",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": null,
+    "rank": 1500,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Entries in this dictionary will only be redacted in this dossier",
+    "addToDictionaryAction": false,
+    "label": "Dossier Redaction",
+    "hasDictionary": true,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  }
+]
@@ -86,3 +86,4 @@ Toxicol Sci
 Toxicol Sci.
 Toxicol Sci. 1
 Test Ignored Hint Published Information
+Workshop
@@ -0,0 +1,12 @@
+{
+  "defaultColor": "#9398a0",
+  "requestAdd": "#04b093",
+  "requestRemove": "#04b093",
+  "notRedacted": "#c498fa",
+  "analysisColor": "#dd4d50",
+  "updatedColor": "#fdbd00",
+  "dictionaryRequestColor": "#5b97db",
+  "manualRedactionColor": "#9398a0",
+  "previewColor": "#9398a0",
+  "ignoredHintColor": "#e7d4ff"
+}
@@ -0,0 +1,21 @@
+5 Batch
+Author
+Batch Analysis
+Certificate of Analysis
+Chemical Investigator
+Contributors
+Five Batch
+GLP Certificate
+Pathologist
+Performing Laboratory
+Principle Investigator
+Quality Assurance
+References
+Report of Peer Review
+Study Director
+Study Monitor
+Test Facility
+Test Site
+Trial Site
+altitude
+latitude
@@ -0,0 +1,234 @@
+. Comparative Biochemistry and Physiology Part C: Toxicology & Pharmacology C Toxicology & Pharmacology
+. J. Exp. Zool
+. Lewis Publ., Boca Raton,
+1944; 82:377-390
+7th ed.; The Iowa State University Press: Ames, IA
+7th ed.; The Joint Nature ConservationIowa State University Press: Ames, IAReproduction, Fertility and Development 2
+ATLA
+Academic Press
+Advances in
+Ageing Dev.
+Agromedicine
+Am J Epidemiol
+Am J Ind Med
+Am J Med Genet A
+Am J Respir Crit Care Med
+American Society for Testing and Materials, Philadelphia, Pennsylvania.
+Anal Chem.
+Anal, Chem.,
+Animal Reproduction Science
+Annals of
+Annals of Maths Stats
+Apidologie
+Aq. Toxicol.
+Aquatic Ecology
+Aquatic Toxicology
+Aquatic toxicology
+Arch Environ Contam Toxicol
+Arch Toxicol
+Arch Toxicol.
+Archives of Environmental Contamination and Toxicology
+Archives of Toxicology
+Australasian Journal of Ecotoxicology
+BioFactors
+Biomedical chromatography
+Biometrics
+Biometrika
+Birth Defects Res. B. Dev. Reprod. Toxicol
+Birth Defects Res. B. Dev. Reprod. Toxicol.
+Br Med J
+British Health and Safety Executive, Chemicals Regulation Directorate Pesticides
+Brown Company Publishers, Dubuque, Iowa
+Bull Environ Contam Toxicol.
+Bull. Environ. Contam. Toxicol
+Bulletin
+Bulletin of Environmental Contamination and Toxicology
+CHEMOSPHERE
+Cahiers Agricultures
+Cancer Letters
+Cancer Res
+Carcinogenesis
+Chem. Res. Toxicol.
+Chem. Res. in Toxicol.
+Chemosphere
+Chinese Journal of Chromatography
+Clin Anat.
+Comparative Biochemistry and Physiology Part C: Toxicology & Pharmacology C Toxicology & Pharmacology
+Crit Rev Toxicol
+Critical Reviews in Toxicology
+Curr. Med. Chem.
+Current approaches in the statistical analysis of ecotoxicity data: guidance to application
+DEFRA
+Deutsche Lebensmittel-Rundschau
+Deutsche Lebensmittel-Rundschau.
+Dongbei Nongye Daxue Xuebao
+Dual-Gold
+EFSA Journal
+Ecological Research
+Ecotoxicol. Environ. Safety
+Ecotoxicology and Environmental Safety
+Endocrinology
+Env. Mol. Mutagen
+Environ Health Perspect
+Environ Health Perspect.
+Environ Health Perspect. 1
+Environ Mutat
+Environ Sci Pollut Res
+Environ Sci Technol
+Environ Toxicol.
+Environ and Molecular Mutagenesis
+Environ. Health Perspec.
+Environ. Health Perspect.
+Environ. Sci. Technol
+Environ. Sci. Technol.
+Environment International
+Environmental & Molecular Mutagenesis
+Environmental Health
+Environmental Health Perspectives
+Environmental Monitoring and Assessment
+Environmental Pollution
+Environmental Protection Agency Federal Register
+Environmental Science & Technology
+Environmental Science and Technology
+Environmental Science and Technology.
+Environmental Toxicology and Chemistry
+Environmental monitoring and assessment
+Environmental toxicology and chemistry
+Essays in Honor of Harold Hotelling
+Eur Respir
+Experimental Cel] Research
+FUNDAMENTAL AND APPLIED TOXICOLOGY
+Fischer, Jena
+Fish Sci
+Food Cosmet. Toxicol.
+Franckh, Stuttgar
+Franckh, Stuttgart
+Free Rad. Res
+Fundamentals of Applied Toxicology
+Fundamentals of Applied Toxicology1988
+Gen. Comp. Endocrinol
+Gen. Comp.Endocrinol
+Gen.Com. Endocrinol.
+General and Comparative Endocrinology
+Greenhouse Pest Management
+Gustav Fischer Verlag, Stuttgart
+Handbook of water purity and quality.
+Health and Safety Executive, Chemi-cals Regulation Directorate Pesticides
+Herpetologica
+High-Throughput Screening Methods in Toxicity Testing
+High-Throughput Screening Methods in Toxicity Testing. Hoboken, NJ: John Wiley & Sons
+ISBN
+ISSN
+ISSN/ISBN
+Ibis
+Int Arch Occup Environ Health
+Int J Cancer
+Irish Journal of Agricultural and Food Research
+Ist Annual Symposium on Aquatic Toxicology ASTM,
+J Econom
+J Endocrinol
+J LARGILLIERE
+J Radiat Res
+J Steroid Biochem Mol Biol
+J. Invest. Derm
+Joint Nature Conservation Committee
+Journal Pharmacology Experimental Therapeutics
+Journal of
+Journal of Animal Ecology
+Journal of Applied Ecology
+Journal of Applied Entomology
+Journal of Economic Entomology
+Journal of Environmental Science and Health
+Journal of Experimental Biology and Ecology
+Journal of Hazardous Materials
+Journal of Microbiology
+Journal of Toxicology and
+Journal of agricultural and food chemistry
+Journal of the American College of Toxicology
+Journal of the Royal Statistical Society,
+Leaflet
+Limnologica
+Marine Enironmental Research
+Marine Environmental Research
+Marine Pollution Bulletin
+Mass Spectrometry
+Medical Journal
+Middle Atlantic Reproduction and Teratology Association
+Mol Mutagen
+Mol. Cell. Endocrinol.
+Mutagenesis
+Mutat Res
+Mutation Research
+National Academy of Science,
+Nature
+Neurotoxicity research
+Nonparametric Statistics for the Behavioral Sciences
+OECD Guideline for the testing of chemicals
+OECD Guidelines for Testing of Chemicals
+OECD Guidelines for testing of chemicals.
+OECD Principles of Good Laboratory Practice and Compliance Monitoring
+OECD Series on Testing Assessment
+Occup Environ Med
+Occupational and Environmental Medicine
+Office of Prevention, Pesticides and Toxic Substances
+Official Journal of the European Communities
+Oxford University Press
+PLoS ONE
+Pediat. Res
+Pesticide Biochemistry and Physiology
+Physiol Chem
+Planta daninha
+Press
+Principles and Procedures of Statistics
+Principles and Procedures of Statistics, A Biometrical Approach
+Proc Natl Acad Sci USA
+Psychopharmacologia
+Publication
+Publications Office of the European Union, Luxembourg
+Published: Y
+Publisher Gustav Fischer, Stuttgart
+Raven press,
+Reg. Tox. Pharmacol
+Regul. Toxicol. Pharmacol
+Regulatory Toxicology and Pharmacology
+Reproductive BioMedicine Online
+Reproductive Toxicology
+Schweizerbart’sche Verlagsbuchhandlung
+Science of the Total Environment
+Scientific Reports
+Stain Technol
+Superorder Monogonnata
+Swiss Ordinance relating to Good Laboratory Practice
+Teratology
+The American Statistician
+The Chemical Society,
+The Inland Waters Volume
+The Royal Society for the Protection of Birds
+Tidepool Scientific Software, Mckinleyville, CA, USA
+Toxicol Chem
+Toxicol Pathol.
+Toxicol Sci
+Toxicol Sci.
+Toxicol Sci. 1
+Toxicol in Vitro
+Toxicol. Sci,
+Toxicol. Sci.
+Toxicol. in Vitro
+Toxicologic Pathology
+Toxicological Sciences
+Toxicological and Environmental Chemistry
+Toxicology Letters
+Toxicology and Applied Pharmacology
+Toxicology and Industrial Health
+Toxicology in Vitro
+Trends in Neurosciences
+Umweltbundesamt Development & Research Project
+WATER CHLORINATION:CHEM, ENVIRON. IMPACT HEALTH EFF., PROC. CONF
+Water Res.
+Workshop
+World Journal of Pharmaceutical Research
+XIII Symposium Pesticide Chemistry
+edition
+http://www.iobc-wprs.org
+ournal of the Royal Statistical Society
+paper was presented
@@ -0,0 +1,22 @@
+{
+  "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+  "name": "EFSA sanitisation GFL v1",
+  "description": "Redact all authors, laboratory addresses and other geolocation.",
+  "dateAdded": "2021-01-21T14:25:35Z",
+  "dateModified": "2021-07-19T08:09:27Z",
+  "createdBy": "e3aed6ea-a9e5-4f3f-bde7-6f0fe0c4362c",
+  "modifiedBy": "e3aed6ea-a9e5-4f3f-bde7-6f0fe0c4362c",
+  "validFrom": null,
+  "validTo": null,
+  "downloadFileTypes": [
+    "PREVIEW",
+    "REDACTED",
+    "DELTA_PREVIEW"
+  ],
+  "reportTypes": [
+    "EXCEL_MULTI_FILE",
+    "WORD_SINGLE_FILE_APPENDIX_A1_TEMPLATE",
+    "WORD_SINGLE_FILE_APPENDIX_A2_TEMPLATE"
+  ],
+  "dossierTemplateStatus": "ACTIVE"
+}
@@ -0,0 +1,47 @@
+[
+  {
+    "name": "1.1 personal data (incl. geolocation); Article 39(e)(3)",
+    "description": "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)",
+    "reason": "Article 39(e)(3) of Regulation (EC) No 178/2002"
+  },
+  {
+    "name": "1.2 vertebrate study related personal data (incl. geolocation); Article 39(e)(2)",
+    "description": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information",
+    "reason": "Article 39(e)(2) of Regulation (EC) No 178/2002"
+  },
+  {
+    "name": "2. manufacturing or production process",
+    "description": "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety",
+    "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"
+  },
+  {
+    "name": "3. links between a producer and applicant",
+    "description": "commercial links between a producer or importer and the applicant or the authorisation holder, where applicable",
+    "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No  178/2002)"
+  },
+  {
+    "name": "4. commercial information",
+    "description": "commercial information revealing sourcing, market shares or business strategy of the applicant",
+    "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009  (making reference to Article 39 of Regulation EC No 178/2002)"
+  },
+  {
+    "name": "5. quantitative composition",
+    "description": "quantitative composition of the subject matter of the request, except for information which is relevant to the assessment of safety",
+    "reason": "Article 63(2)(a) of Regulation (EC) No  1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"
+  },
+  {
+    "name": "6. specification of impurity",
+    "description": "the specification of impurity of the active substance and the related methods of analysis for impurities in the active substance as manufactured, except for the impurities that are considered to be toxicologically, ecotoxicologically or environmentally relevant and the related methods of analysis for such impurities",
+    "reason": "Article 63(2)(b) of Regulation (EC) No 1107/2009"
+  },
+  {
+    "name": "7. results of production batches",
+    "description": "results of production batches of the active substance including impurities",
+    "reason": "Article 63(2)(c) of Regulation (EC) No 1107/2009"
+  },
+  {
+    "name": "8. composition of a plant protection product",
+    "description": "information on the complete composition of a plant protection product",
+    "reason": "Article 63(2)(d) of Regulation (EC) No 1107/2009"
+  }
+]
@@ -0,0 +1,480 @@
+package drools
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+
+global Section section
+
+
+// --------------------------------------- CBI rules -------------------------------------------------------------------
+rule "0: Add CBI_author from ai"
+    when
+        Section(aiMatchesType("CBI_author"))
+    then
+        section.addAiEntities("CBI_author", "CBI_author");
+    end
+
+
+rule "0: Combine ai types CBI_author from ai"
+    when
+        Section(aiMatchesType("ORG"))
+    then
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+    end
+
+
+rule "1: Redact CBI Authors (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "2: Redact CBI Authors (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "3: Redact not CBI Address (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
+        section.ignoreRecommendations("CBI_address");
+    end
+
+
+rule "4: Redact CBI Address (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "5: Do not redact genitive CBI_author"
+    when
+        Section(matchesType("CBI_author"))
+    then
+        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
+    end
+
+
+rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "13: Redact addresses that start with BL or CTL"
+    when
+        Section(searchText.contains("BL") || searchText.contains("CT"))
+    then
+        section.redactNotAndRecommendByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address", 13, "Laboratory for vertebrate studies found");
+    end
+
+
+rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "16: Add recommendation for Addresses in Test Organism sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
+    then
+        section.recommendLineAfter("Source:", "CBI_address");
+    end
+
+
+rule "17: Add recommendation for Addresses in Test Animals sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
+    then
+        section.recommendLineAfter("Source", "CBI_address");
+    end
+
+
+rule "18: Do not redact Names and Addresses if Published Information found"
+    when
+        Section(matchesType("published_information"))
+    then
+        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
+        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
+    end
+
+
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+
+rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 19, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 20, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "21: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "22: Redact Emails by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "23: Redact telephone numbers by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true))
+    then
+        section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 23, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "24: Redact telephone numbers by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true))
+    then
+        section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 24, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "25: Recommend telephone numbers by RegEx"
+    when
+        Section(containsRegEx("\\d{3,}", true) || containsRegEx("[+]\\d{2,}", true))
+    then
+        section.addRecommendationByRegEx("([+]\\d{2,3} (\\d{7,12})\\b|((([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|(\\(?\\b(\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b))", true, 1, "PII");
+    end
+
+
+rule "26: Redact contact information (Non vertebrate study)"
+    when
+       Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+       || text.contains("Phone:")
+       || text.contains("Fax:")
+       || text.contains("Tel.:")
+       || text.contains("Tel:")
+       || text.contains("E-mail:")
+       || text.contains("Email:")
+       || text.contains("e-mail:")
+       || text.contains("E-mail address:")
+       || text.contains("Alternative contact:")
+       || text.contains("Telephone number:")
+       || text.contains("Telephone No:")
+       || text.contains("Fax number:")
+       || text.contains("Telephone:")
+       || text.contains("Phone No.")
+       || text.contains("European contact:")))
+    then
+          section.redactLineAfter("Contact point:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+     end
+
+
+rule "27: Redact contact information (Vertebrate study)"
+    when
+       Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+       || text.contains("Phone:")
+       || text.contains("Fax:")
+       || text.contains("Tel.:")
+       || text.contains("Tel:")
+       || text.contains("E-mail:")
+       || text.contains("Email:")
+       || text.contains("e-mail:")
+       || text.contains("E-mail address:")
+       || text.contains("Alternative contact:")
+       || text.contains("Telephone number:")
+       || text.contains("Telephone No:")
+       || text.contains("Fax number:")
+       || text.contains("Telephone:")
+       || text.contains("Phone No.")
+       || text.contains("European contact:")))
+    then
+          section.redactLineAfter("Contact point:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+     end
+
+
+rule "28: Redact contact information if applicant is found (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")))
+    then
+        section.redactLineAfter("Contact point:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "29: Redact contact information if applicant is found (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")))
+    then
+        section.redactLineAfter("Contact point:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "30: Redact contact information if Producer is found (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")))
+    then
+        section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "31: Redact contact information if Producer is found (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")))
+    then
+        section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "32: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:"))
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 32, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "33: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:"))
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 33, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "34: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:"))
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 34, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "35: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:"))
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 35, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "36: Redact PERFORMING LABORATORY (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:"))
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 36, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactNot("CBI_address", 36, "Performing laboratory found for non vertebrate study");
+    end
+
+
+rule "37: Redact PERFORMING LABORATORY (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:"))
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 37, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "50: Purity Hint"
+    when
+        Section(searchText.toLowerCase().contains("purity"))
+    then
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+    end
+
+
+rule "51: Ignore dossier_redaction entries if confidential"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redaction"));
+    then
+        section.ignore("dossier_redaction");
+    end
+
+
+rule "52: Redact signatures (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 52, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "53: Redact signatures (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 53, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "54: Redact Logos (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
+    then
+        section.redactImage("logo", 54, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
@@ -0,0 +1,185 @@
+{
+  "types": [
+    {
+      "type": "CBI_address",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 140,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "All site names and addresses, and location (e.g. Syngenta, Monthey, GPS Co-ordinates, Mr Smith of … providing the…). Except addresses in published literature and the applicant address.",
+      "addToDictionaryAction": true,
+      "label": "CBI Address",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "CBI_author",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 130,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "All authors named in the study documentation. Except names in published literature.",
+      "addToDictionaryAction": true,
+      "label": "CBI Author",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "PII",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 150,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "Not authors but listed in the document: Names, signatures, telephone, email etc.; e.g. Reg Manager, QA Manager",
+      "addToDictionaryAction": true,
+      "label": "PII",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "formula",
+      "hexColor": "#036ffc",
+      "recommendationHexColor": "#8df06c",
+      "rank": 1002,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure formula colors.",
+      "addToDictionaryAction": false,
+      "label": "Formula",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "hint_only",
+      "hexColor": "#fa98f7",
+      "recommendationHexColor": "#8df06c",
+      "rank": 50,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Entries of this dictionary will be highlighted only",
+      "addToDictionaryAction": false,
+      "label": "Hint Only",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "image",
+      "hexColor": "#bdd6ff",
+      "recommendationHexColor": "#8df06c",
+      "rank": 999,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure image colors.",
+      "addToDictionaryAction": false,
+      "label": "Image",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "logo",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 1001,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure logo colors.",
+      "addToDictionaryAction": false,
+      "label": "Logo",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "must_redact",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 100,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Entries of this dictionary get redacted wherever found.",
+      "addToDictionaryAction": false,
+      "label": "Must Redact",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "ocr",
+      "hexColor": "#bdd6ff",
+      "recommendationHexColor": "#8df06c",
+      "rank": 1000,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure ocr colors.",
+      "addToDictionaryAction": false,
+      "label": "Ocr",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "published_information",
+      "hexColor": "#85ebff",
+      "recommendationHexColor": "#8df06c",
+      "rank": 70,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "Manual managed list of public journals and papers that need no redaction",
+      "addToDictionaryAction": true,
+      "label": "Published Information",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "signature",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 1003,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure signature colors.",
+      "addToDictionaryAction": false,
+      "label": "Signature",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "imported_redaction",
+      "hexColor": "#f0f0c0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 9999,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "Redaction Annotations that were imported from documents",
+      "addToDictionaryAction": false,
+      "label": "Imported Redaction",
+      "hasDictionary": false,
+      "systemManaged": true,
+      "autoHideSkipped": true
+    }
+  ]
+}
Author	SHA1	Message	Date
Philipp Schramm	c1192ceefd	RED-3816: Bugfix with adding images to sections	2022-05-30 13:52:57 +02:00
Timo Bejan	77584c9a5a	Pull request #394 : RED-3800 String Performance matching test Merge in RED/redaction-service from RED-3800-string-performance-test to master * commit '21d717f0837c9cc9b2f5b251d372db88b25a7b6d': RED-3800 String Performance matching test RED-3800 String Performance matching test	2022-05-24 11:09:08 +02:00
Timo Bejan	21d717f083	RED-3800 String Performance matching test	2022-05-24 12:01:10 +03:00
Timo Bejan	c85ce25ed4	RED-3800 String Performance matching test	2022-05-24 11:58:34 +03:00
Philipp Schramm	f608d991a3	Pull request #393 : RED-3816: Implemented ThenAction to redact section with rectangle Merge in RED/redaction-service from RED-3816 to master * commit 'eb7137315208dadf3a679e21b05ec16d53c4ec43': RED-3816: Implemented ThenAction to redact section with rectangle	2022-05-24 09:00:07 +02:00
Ali Oezyetimoglu	e914b09d1e	Pull request #392 : RED-3674: Fixed force redactions overrides idRemoval with different id Merge in RED/redaction-service from RED-3674-rs1 to master * commit '87d64b6d1285eaf7cd5ee3c2fb899f92063bc70f': RED-3674: Fixed force redactions overrides idRemoval with different id	2022-05-23 18:48:10 +02:00
aoezyetimoglu	87d64b6d12	RED-3674: Fixed force redactions overrides idRemoval with different id	2022-05-23 17:38:23 +02:00
Philipp Schramm	eb71373152	Merge branch 'master' into RED-3816	2022-05-23 16:27:12 +02:00
Philipp Schramm	a65e23e45d	RED-3816: Implemented ThenAction to redact section with rectangle	2022-05-23 16:26:47 +02:00
Dominique Eiflaender	4aaef260d7	Pull request #390 : RED-4075: Add legalBasis if Rule Entity overrides Ai Recommendation Merge in RED/redaction-service from RED-4075 to master * commit 'c669f9a09a3e5e254d41715f65fef3b158873338': RED-4075: Add legalBasis if Rule Entity overrides Ai Recommendation	2022-05-23 14:05:38 +02:00
deiflaender	c669f9a09a	RED-4075: Add legalBasis if Rule Entity overrides Ai Recommendation	2022-05-23 13:54:27 +02:00
deiflaender	b1bad03090	RED-4014: Changed metrics config	2022-05-12 16:08:54 +02:00
deiflaender	0a0d9fe31a	RED-4014: Added names to @Timed annotations	2022-05-12 15:56:03 +02:00
deiflaender	fd37e3deab	RED-4014: Added @Timed annotations to AnaylseService	2022-05-12 15:10:35 +02:00
Dominique Eiflaender	7fb04cfa48	Pull request #389 : RED-4010: Fixed problem with strange unicode in json d835 Merge in RED/redaction-service from RED-4010 to master * commit '0f5f7974cc46e36d39d514ad03b5adcf062c9526': RED-4010: Fixed problem with strange unicode in json d835	2022-05-11 14:41:07 +02:00
deiflaender	0f5f7974cc	RED-4010: Fixed problem with strange unicode in json d835	2022-05-11 14:38:06 +02:00
Dominique Eiflaender	0601ec4862	Pull request #388 : RED-4009: Fixed NPE if comments are null in RedactionLogMergeService Merge in RED/redaction-service from RED-4009 to master * commit 'd2807e0ff1ef288c7456383974a8607a8fca4821': RED-4009: Fixed NPE if comments are null in RedactionLogMergeService	2022-05-10 13:24:23 +02:00
deiflaender	d2807e0ff1	RED-4009: Fixed NPE if comments are null in RedactionLogMergeService	2022-05-10 13:20:57 +02:00
Timo Bejan	e49bbfca88	Pull request #387 : RED-3800 switched jackson for dsl-json for internal non-rest related operations Merge in RED/redaction-service from RED-3800-dsljson to master * commit '7967453c676094efdc438eea1cca2b9e00856c0d': RED-3800 api cleanup RED-3800 api cleanup RED-3800 switched jackson for dsl-json for internal non-rest related operations	2022-05-09 21:14:04 +02:00
Timo Bejan	7967453c67	RED-3800 api cleanup	2022-05-09 22:10:53 +03:00
Timo Bejan	a3d002929d	RED-3800 api cleanup	2022-05-09 22:03:33 +03:00
Timo Bejan	8a44d6299c	RED-3800 switched jackson for dsl-json for internal non-rest related operations	2022-05-09 22:00:01 +03:00
Timo Bejan	78de7371a0	Pull request #385 : RED-3946 - quotation-mark fix Merge in RED/redaction-service from RED-3946-qf to master * commit 'df2fcd2e5e48298131ebf7a00bf2ed80a9fc2d96': RED-3946 - quotation-mark fix	2022-05-09 14:47:03 +02:00
Dominique Eiflaender	924f791e4d	Pull request #386 : RED-3959: Do not remove entities on larger false_recommendactions Merge in RED/redaction-service from RED-3959 to master * commit '542bbb294a27bf2eb2832a144d239b820bbc0b5f': RED-3959: Do not remove entities on larger false_recommendactions	2022-05-09 12:34:13 +02:00
deiflaender	542bbb294a	RED-3959: Do not remove entities on larger false_recommendactions	2022-05-09 12:27:36 +02:00
Timo Bejan	df2fcd2e5e	RED-3946 - quotation-mark fix	2022-05-09 12:14:09 +03:00
Timo Bejan	8e819a0d26	Pull request #384 : RED-3992 Merge in RED/redaction-service from RED-3992 to master * commit '01e7115d301d8706962390b76af0769eab67c1fe': RED-3992 - Fixed PMD RED-3992 - Replaced naive approach with aho corassick string search. Cleaned up code. reduced ammount of unnecesary conversions/invocations RED-3992 - Replaced naive approach with aho corassick string search. Cleaned up code. reduced ammount of unnecesary conversions/invocations RED-3800 Exception handling for processed files	2022-05-09 10:05:50 +02:00
Timo Bejan	01e7115d30	RED-3992 - Fixed PMD	2022-05-09 09:02:11 +03:00
Timo Bejan	747323f882	RED-3992 - Replaced naive approach with aho corassick string search. Cleaned up code. reduced ammount of unnecesary conversions/invocations	2022-05-09 08:44:42 +03:00
Timo Bejan	729a7334d9	Merge branch 'release/3.107.x' into test-performance	2022-05-06 10:28:47 +03:00
Timo Bejan	36967cab2c	RED-3992 - Replaced naive approach with aho corassick string search. Cleaned up code. reduced ammount of unnecesary conversions/invocations	2022-05-06 10:28:12 +03:00
Timo Bejan	cfd9d7665b	Pull request #383 : RED-3946 - added alternative quatation marks as separators Merge in RED/redaction-service from RED-3946 to master * commit '23d84980cd1528b1276e28aab81bb5bf0a682256': RED-3946 - added alternative quatation marks as separators	2022-05-03 17:15:12 +02:00
Timo Bejan	23d84980cd	RED-3946 - added alternative quatation marks as separators	2022-05-03 18:12:07 +03:00
Timo Bejan	314f7ebdd9	Pull request #382 : Pdftron api cleanup Merge in RED/redaction-service from pdftron-api-cleanup to master * commit 'f666a0330161daed8d3f59189bbe258327926995': RED-3800 code cleanup RED-3800 code cleanup	2022-05-03 12:26:22 +02:00
Timo Bejan	f666a03301	RED-3800 code cleanup	2022-05-03 13:23:10 +03:00
Timo Bejan	313ec32d30	RED-3800 code cleanup	2022-05-03 13:04:00 +03:00
Timo Bejan	76dca41299	Pull request #381 : RED-3915 - source id Merge in RED/redaction-service from RED-3915 to master * commit 'f21e48e3d75c78e0ce5cf92a614e92e9c721eede': RED-3915 - source id	2022-04-25 15:03:29 +02:00
Timo Bejan	f21e48e3d7	RED-3915 - source id	2022-04-25 15:46:12 +03:00