RED-6224: Made dictionary cache multitenancy ready

Merge in RED/redaction-service from RED-6250 to master

* commit 'fe5121b1edf784d5cc57abd9cd8e27f70dea0f4a':
  RED-6250: Switched to new metrics-commons version to not have to instantiate a factory (reduces code duplication)
  RED-6250: Changed function-timer metric to use code from metric commons

bamboo-specs/src/main/resources/scripts/sonar-java.sh

@@ -12,7 +12,8 @@ mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
 echo "dependency-check:aggregate"
 mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    org.owasp:dependency-check-maven:aggregate
+    org.owasp:dependency-check-maven:aggregate \
+    -DknownExploitedEnabled=false
 
 if [[ -z "${bamboo_repository_pr_key}" ]]
 then

pom.xml

@@ -7,7 +7,7 @@
 
     <artifactId>redaction-service</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>3.0-SNAPSHOT</version>
+    <version>4.0-SNAPSHOT</version>
 
 
     <packaging>pom</packaging>

redaction-service-v1/pom.xml

@@ -12,7 +12,7 @@
 
     <artifactId>redaction-service-v1</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>1.0-SNAPSHOT</version>
+    <version>4.0-SNAPSHOT</version>
 
     <packaging>pom</packaging>
 
@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.20.0</version>
+                <version>1.22.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -6,13 +6,13 @@
     <parent>
         <artifactId>redaction-service-v1</artifactId>
         <groupId>com.iqser.red.service</groupId>
-        <version>1.0-SNAPSHOT</version>
+        <version>4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.299.0</persistence-service.version>
+        <persistence-service.version>2.1.0</persistence-service.version>
     </properties>
 
     <dependencies>
@@ -32,13 +32,18 @@
 
         <dependency>
             <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-api-v1</artifactId>
+            <artifactId>persistence-service-internal-api-v1</artifactId>
             <version>${persistence-service.version}</version>
             <exclusions>
+
                 <exclusion>
                     <groupId>com.iqser.red.service</groupId>
                     <artifactId>redaction-service-api-v1</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>persistence-service-api-v1</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java

@@ -1,40 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class AnalyzeRequest {
-
-    private MessageType messageType;
-
-    private String dossierId;
-    private String fileId;
-    private String dossierTemplateId;
-    private ManualRedactions manualRedactions;
-    private OffsetDateTime lastProcessed;
-    private int analysisNumber;
-
-    @Builder.Default
-    private Set<Integer> excludedPages = new HashSet<>();
-    @Builder.Default
-    private Set<Integer> sectionsToReanalyse = new HashSet<>();
-
-    @Builder.Default
-    private List<FileAttribute> fileAttributes = new ArrayList<>();
-
-}
-

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java

@@ -1,40 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.Set;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class AnalyzeResult {
-
-    private MessageType messageType;
-
-    private String dossierId;
-    private String fileId;
-    private long duration;
-    private int numberOfPages;
-    private boolean hasUpdates;
-    private long dictionaryVersion;
-    private long dossierDictionaryVersion;
-    private long rulesVersion;
-    private long legalBasisVersion;
-
-    private boolean wasReanalyzed;
-
-    private int analysisVersion;
-    private int analysisNumber;
-
-    private ManualRedactions manualRedactions;
-
-    private Set<FileAttribute> addedFileAttributes;
-
-}
-

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/CellRectangle.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class CellRectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java

@@ -1,20 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class Change {
-
-    private int analysisNumber;
-    private ChangeType type;
-    private OffsetDateTime dateTime;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java

@@ -1,7 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum ChangeType {
-    ADDED,
-    REMOVED,
-    CHANGED
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Engine.java

@@ -1,7 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum Engine {
-    DICTIONARY,
-    NER,
-    RULE
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/FileAttribute.java

@@ -1,19 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class FileAttribute {
-
-    private String id;
-    private String label;
-    private String placeholder;
-    private String value;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedaction.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImportedRedaction {
-
-    private String id;
-
-    @Builder.Default
-    private List<Rectangle> positions = new ArrayList<>();
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedactions.java

@@ -1,24 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@Data
-@Builder
-@CompiledJson
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImportedRedactions {
-
-    @Builder.Default
-    private Map<Integer, List<ImportedRedaction>> importedRedactions = new HashMap<>();
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java

@@ -1,59 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-import java.util.HashMap;
-import java.util.Map;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Builder
-public class ManualChange {
-
-    private AnnotationStatus annotationStatus;
-    private ManualRedactionType manualRedactionType;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime requestedDate;
-    private String userId;
-    private Map<String, String> propertyChanges = new HashMap<>();
-
-
-    public static ManualChange from(BaseAnnotation baseAnnotation) {
-
-        ManualChange manualChange = new ManualChange();
-        manualChange.annotationStatus = baseAnnotation.getStatus();
-        manualChange.processedDate = baseAnnotation.getProcessedDate();
-        manualChange.requestedDate = baseAnnotation.getRequestDate();
-        manualChange.userId = baseAnnotation.getUser();
-        return manualChange;
-    }
-
-
-    public boolean isProcessed() {
-
-        return processedDate != null;
-    }
-
-
-    public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        this.manualRedactionType = manualRedactionType;
-        return this;
-    }
-
-
-    public ManualChange withChange(String property, String value) {
-
-        this.propertyChanges.put(property, value);
-        return this;
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java

@@ -1,13 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum ManualRedactionType {
-    ADD_LOCALLY,
-    ADD_TO_DICTIONARY,
-    REMOVE_LOCALLY,
-    REMOVE_FROM_DICTIONARY,
-    FORCE_REDACT,
-    FORCE_HINT,
-    RECATEGORIZE,
-    LEGAL_BASIS_CHANGE,
-    RESIZE
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MessageType.java

@@ -1,10 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum MessageType {
-
-    ANALYSE,
-    REANALYSE,
-    STRUCTURE_ANALYSE,
-    SURROUNDING_TEXT
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java

@@ -1,15 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class Point {
-
-    private float x;
-    private float y;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ReanalyzeResult.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ReanalyzeResult {
-
-    private RedactionLog redactionLog;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class Rectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-
-    private int page;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java

@@ -1,37 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@Data
-@CompiledJson
-@AllArgsConstructor
-@NoArgsConstructor
-public class RedactionLog {
-
-    /**
-     * Version 0 Redaction Logs have manual redactions merged inside them
-     * Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime.
-     */
-    private long analysisVersion;
-
-    /**
-     * Which analysis created this redactionLog.
-     */
-    private int analysisNumber;
-
-    private List<RedactionLogEntry> redactionLogEntry = new ArrayList<>();
-    private List<RedactionLogLegalBasis> legalBasis = new ArrayList<>();
-
-    private long dictionaryVersion = -1;
-    private long dossierDictionaryVersion = -1;
-    private long rulesVersion = -1;
-    private long legalBasisVersion = -1;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class RedactionLogChanges {
-
-    private RedactionLog redactionLog;
-    private boolean hasChanges;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogComment.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionLogComment {
-
-    private long id;
-    private String user;
-    private String text;
-    private String annotationId;
-    private String fileId;
-    private OffsetDateTime date;
-    private OffsetDateTime softDeletedTime;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -1,96 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-
-import lombok.*;
-
-import java.util.*;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@EqualsAndHashCode
-public class RedactionLogEntry {
-
-    private String id;
-    private String type;
-    private String value;
-    private String reason;
-    private int matchedRule;
-    private boolean rectangle;
-    private String legalBasis;
-
-    private boolean imported;
-
-    private boolean redacted;
-    private boolean isHint;
-    private boolean isRecommendation;
-    private boolean isFalsePositive;
-
-    private String section;
-    private float[] color;
-
-    @Builder.Default
-    private List<Rectangle> positions = new ArrayList<>();
-    private int sectionNumber;
-
-    private String textBefore;
-    private String textAfter;
-
-    @Builder.Default
-    private List<RedactionLogComment> comments = new ArrayList<>();
-
-    private int startOffset;
-    private int endOffset;
-
-    private boolean isImage;
-    private boolean imageHasTransparency;
-
-    private boolean isDictionaryEntry;
-    private boolean isDossierDictionaryEntry;
-
-    private boolean excluded;
-
-    private String sourceId;
-
-    @EqualsAndHashCode.Exclude
-    @Builder.Default
-    private List<Change> changes = new ArrayList<>();
-
-    @EqualsAndHashCode.Exclude
-    @Builder.Default
-    private List<ManualChange> manualChanges = new ArrayList<>();
-
-    private Set<Engine> engines = new HashSet<>();
-
-    private Set<String> reference = new HashSet<>();
-
-    @Builder.Default
-    private Set<String> importedRedactionIntersections = new HashSet<>();
-
-
-    public boolean lastChangeIsRemoved() {
-
-        return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
-    }
-
-
-    public boolean isLocalManualRedaction() {
-
-        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
-    }
-
-
-    public boolean isManuallyRemoved() {
-
-        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
-    }
-
-
-    private <T> Optional<T> last(List<T> list) {
-
-        return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogLegalBasis.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionLogLegalBasis {
-
-    private String name;
-    private String description;
-    private String reason;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java

@@ -1,34 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionRequest {
-
-    private String dossierId;
-    private String fileId;
-    private String dossierTemplateId;
-    private ManualRedactions manualRedactions;
-    @Builder.Default
-    private Set<Integer> excludedPages = new HashSet<>();
-
-    private Colors colors;
-    private List<Type> types;
-
-    private boolean includeFalsePositives;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionResult {
-
-    private byte[] document;
-    private int numberOfPages;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java

@@ -1,34 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class SectionArea {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-    private int page;
-    private String header;
-
-
-    public boolean contains(Rectangle other) {
-
-        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft()
-                .getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft()
-                .getY() + other.getHeight();
-    }
-
-
-    // TODO we should only use one rectangle class.
-    public boolean contains(com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle other) {
-
-        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeftX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeftX() + other.getWidth() && this.getTopLeft()
-                .getY() <= other.getTopLeftY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeftY() + other.getHeight();
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java

@@ -1,33 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.*;
-
-@Data
-@CompiledJson
-@AllArgsConstructor
-@NoArgsConstructor
-public class SectionGrid {
-
-    private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();
-
-    private List<SectionGridSection> sections = new ArrayList<>();
-
-    @Data
-    @NoArgsConstructor
-    @AllArgsConstructor
-    public static class SectionGridSection {
-
-        private int sectionNumber;
-        private String headline;
-        private Set<Integer> pages;
-        private List<SectionArea> sectionAreas;
-
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.List;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class SectionRectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-    private int part;
-    private int numberOfParts;
-
-    private List<CellRectangle> tableCells;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -1,38 +1,12 @@
 package com.iqser.red.service.redaction.v1.resources;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-
 import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
 public interface RedactionResource {
 
-    @PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult classify(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/debug/sections", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult sections(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
-
-
     @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
     void testRules(@RequestBody String rules);
 
-
-    @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/manual/surrounding-text/{dossierId}/{fileId}", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
-    ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions);
-
 }

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <artifactId>redaction-service-v1</artifactId>
         <groupId>com.iqser.red.service</groupId>
-        <version>1.0-SNAPSHOT</version>
+        <version>4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>redaction-service-server-v1</artifactId>
@@ -93,6 +93,7 @@
         <dependency>
             <groupId>com.iqser.red.commons</groupId>
             <artifactId>metric-commons</artifactId>
+            <version>1.7.0</version>
         </dependency>
         <!-- other external -->
         <dependency>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -9,14 +9,16 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig;
+import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
+import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyWebConfiguration;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
 
-@Import({DefaultWebMvcConfiguration.class})
+@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class, MetricsConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/MetricsConfiguration.java

@@ -0,0 +1,26 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+import com.iqser.gin4.commons.metrics.MetricCommonsConfiguration;
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerFactory;
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
+
+@Configuration
+@Import({MetricCommonsConfiguration.class})
+public class MetricsConfiguration {
+
+    private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
+
+
+    @Bean
+    @Qualifier(REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME)
+    public FunctionTimerValues redactmanagerAnalyzePagewise(FunctionTimerFactory factory) {
+
+        return factory.create(REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
 
 import lombok.Data;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java

@@ -4,8 +4,8 @@ import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -5,7 +5,7 @@ import java.util.regex.Pattern;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.classification.utils;
 
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 
 import lombok.experimental.UtilityClass;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/DictionaryClient.java

@@ -2,9 +2,9 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.DictionaryResource;
 
 @FeignClient(name = "DictionaryResource", url = "${persistence-service.url}")
 public interface DictionaryClient extends DictionaryResource {
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client;
-
-import org.springframework.cloud.openfeign.FeignClient;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PostMapping;
-
-import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
-
-@FeignClient(name = "EntityRecognitionClient", url = "${entity-recognition-service.url}")
-public interface EntityRecognitionClient {
-
-    @PostMapping(value = "/find_authors", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    NerEntities findAuthors(EntityRecognitionRequest entityRecognitionRequest);
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/FileStatusProcessingUpdateClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;
 
 @FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
 public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/LegalBasisClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.LegalBasisMappingResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.LegalBasisMappingResource;
 
 @FeignClient(name = "LegalBasisMappingResource", url = "${persistence-service.url}")
 public interface LegalBasisClient extends LegalBasisMappingResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/RulesClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.RulesResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.RulesResource;
 
 @FeignClient(name = "RulesResource", url = "${persistence-service.url}")
 public interface RulesClient extends RulesResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/TenantsClient.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
+
+@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
+public interface TenantsClient extends TenantsResource {
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client.model;
-
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class EntityRecognitionRequest {
-
-    private List<EntityRecognitionSection> data;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java

@@ -1,21 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client.model;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class EntityRecognitionResult {
-
-    @Builder.Default
-    private Map<Integer, List<EntityRecogintionEntity>> entities = new HashMap<>();
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,31 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -35,100 +15,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
 
-    private final PdfVisualisationService pdfVisualisationService;
     private final DroolsExecutionService droolsExecutionService;
-    private final PdfSegmentationService pdfSegmentationService;
-    private final RedactionStorageService redactionStorageService;
-    private final RedactionLogMergeService redactionLogMergeService;
-    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
-
-
-    @Override
-    public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
-
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                redactionRequest.getFileId(),
-                FileType.ORIGIN));
-        try {
-            Document classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-
-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
-                pdDocument.setAllSecurityToBeRemoved(true);
-
-                pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
-
-                return convert(pdDocument, classifiedDoc.getPages().size());
-
-            } catch (IOException e) {
-                throw new RedactionException(e);
-            }
-
-        } catch (IOException e) {
-            throw new RedactionException(e);
-        }
-
-    }
-
-
-    @Override
-    public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
-
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                redactionRequest.getFileId(),
-                FileType.ORIGIN));
-        try {
-            Document classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-
-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
-                pdDocument.setAllSecurityToBeRemoved(true);
-
-                pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
-                return convert(pdDocument, classifiedDoc.getPages().size());
-
-            } catch (IOException e) {
-                throw new RedactionException(e);
-            }
-
-        } catch (IOException e) {
-            throw new RedactionException(e);
-        }
-
-    }
-
-
-    @Override
-    public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
-
-        Document classifiedDoc;
-
-        try {
-            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-        } catch (Exception e) {
-            throw new RedactionException(e);
-        }
-
-        StringBuilder sb = new StringBuilder();
-        for (Page page : classifiedDoc.getPages()) {
-            for (AbstractTextContainer textContainer : page.getTextBlocks()) {
-                if (textContainer instanceof Table) {
-                    Table table = (Table) textContainer;
-                    sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
-                }
-            }
-        }
-
-        return RedactionResult.builder().document(sb.toString().getBytes()).build();
-
-    }
 
 
     @Override
@@ -141,30 +28,4 @@ public class RedactionController implements RedactionResource {
         }
     }
 
-
-    @Override
-    public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
-
-        return redactionLogMergeService.provideRedactionLog(redactionRequest);
-    }
-
-
-    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
-
-        try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
-            document.save(byteArrayOutputStream);
-            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
-        }
-
-    }
-
-
-    @Override
-    public ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions) {
-
-        var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
-        log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());
-        return result.getManualRedactions();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/AsyncConfig.java

@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import java.util.concurrent.Executor;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+@Configuration
+public class AsyncConfig extends AsyncConfigurerSupport {
+
+    @Override
+    public Executor getAsyncExecutor() {
+
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+
+        executor.setCorePoolSize(7);
+        executor.setMaxPoolSize(42);
+        executor.setQueueCapacity(11);
+        executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
+        executor.setTaskDecorator(new TenantAwareTaskDecorator());
+        executor.initialize();
+
+        return executor;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/EncryptionDecryptionService.java

@@ -0,0 +1,105 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
+import java.security.spec.KeySpec;
+import java.util.Base64;
+
+import javax.annotation.PostConstruct;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import lombok.SneakyThrows;
+
+@Service
+public class EncryptionDecryptionService {
+
+    @Value("${redaction-service.crypto.key:redaction}")
+    private String key;
+
+    private SecretKey secretKey;
+    private byte[] iv;
+
+
+    @SneakyThrows
+    @PostConstruct
+    protected void postConstruct() {
+
+        SecureRandom secureRandom = new SecureRandom();
+        iv = new byte[12];
+        secureRandom.nextBytes(iv);
+        secretKey = generateSecretKey(key, iv);
+    }
+
+
+    @SneakyThrows
+    public String encrypt(String strToEncrypt) {
+
+        return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
+    }
+
+
+    @SneakyThrows
+    public String decrypt(String strToDecrypt) {
+
+        byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
+        return new String(decrypt(bytes), StandardCharsets.UTF_8);
+    }
+
+
+    @SneakyThrows
+    public byte[] encrypt(byte[] data) {
+
+        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+        GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
+        cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
+        byte[] encryptedData = cipher.doFinal(data);
+        ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
+        byteBuffer.putInt(iv.length);
+        byteBuffer.put(iv);
+        byteBuffer.put(encryptedData);
+        return byteBuffer.array();
+    }
+
+
+    @SneakyThrows
+    public byte[] decrypt(byte[] encryptedData) {
+
+        ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
+        int noonceSize = byteBuffer.getInt();
+        if (noonceSize < 12 || noonceSize >= 16) {
+            throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
+        }
+        byte[] iv = new byte[noonceSize];
+        byteBuffer.get(iv);
+
+        SecretKey secretKey = generateSecretKey(key, iv);
+
+        byte[] cipherBytes = new byte[byteBuffer.remaining()];
+        byteBuffer.get(cipherBytes);
+
+        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+        GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
+        cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
+        return cipher.doFinal(cipherBytes);
+    }
+
+
+    @SneakyThrows
+    public SecretKey generateSecretKey(String password, byte[] iv) {
+
+        KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
+        SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
+        byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
+        return new SecretKeySpec(key, "AES");
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/ForwardTenantInterceptor.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Component;
+
+import feign.RequestInterceptor;
+import feign.RequestTemplate;
+
+@Component
+public class ForwardTenantInterceptor implements RequestInterceptor {
+
+    public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
+
+    @Override
+    public void apply(RequestTemplate template) {
+        template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java

@@ -0,0 +1,49 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+
+import static com.iqser.red.service.redaction.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME;
+
+import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.config.BeanPostProcessor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class MultiTenancyMessagingConfiguration {
+
+    @Bean
+    public static BeanPostProcessor multitenancyBeanPostProcessor() {
+
+        return new BeanPostProcessor() {
+
+            @Override
+            public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
+
+                if (bean instanceof RabbitTemplate) {
+
+                    ((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
+                        m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId());
+                        return m;
+                    });
+
+                } else if (bean instanceof AbstractRabbitListenerContainerFactory) {
+
+                    ((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
+                        String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME);
+
+                        if (tenant != null) {
+                            TenantContext.setTenantId(tenant);
+                        } else {
+                            throw new RuntimeException("No Tenant is set queue message");
+                        }
+                        return m;
+                    });
+                }
+                return bean;
+            }
+        };
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyWebConfiguration.java

@@ -0,0 +1,28 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
+
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+
+@Configuration
+public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
+
+    private final TenantInterceptor tenantInterceptor;
+
+
+    @Autowired
+    public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
+
+        this.tenantInterceptor = tenantInterceptor;
+    }
+
+
+    @Override
+    public void addInterceptors(InterceptorRegistry registry) {
+
+        registry.addWebRequestInterceptor(tenantInterceptor);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/StorageConnectionProviderImpl.java

@@ -0,0 +1,45 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.client.TenantsClient;
+import com.iqser.red.storage.commons.model.AzureStorageConnection;
+import com.iqser.red.storage.commons.model.S3StorageConnection;
+import com.iqser.red.storage.commons.service.StorageConnectionProvider;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class StorageConnectionProviderImpl implements StorageConnectionProvider {
+
+    private final TenantsClient tenantsClient;
+    private final EncryptionDecryptionService encryptionDecryptionService;
+
+
+    @Override
+    public AzureStorageConnection getAzureStorageConnection(String tenantId) {
+
+        var tenant = tenantsClient.getTenant(tenantId);
+        return AzureStorageConnection.builder()
+                .connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
+                .containerName(tenant.getAzureStorageConnection().getContainerName())
+                .build();
+    }
+
+
+    @Override
+    public S3StorageConnection getS3StorageConnection(String tenantId) {
+
+        var tenant = tenantsClient.getTenant(tenantId);
+        return S3StorageConnection.builder()
+                .key(tenant.getS3StorageConnection().getKey())
+                .secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
+                .signerType(tenant.getS3StorageConnection().getSignerType())
+                .bucketName(tenant.getS3StorageConnection().getBucketName())
+                .region(tenant.getS3StorageConnection().getRegion())
+                .endpoint(tenant.getS3StorageConnection().getEndpoint())
+                .build();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantAwareTaskDecorator.java

@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.core.task.TaskDecorator;
+import org.springframework.lang.NonNull;
+
+public class TenantAwareTaskDecorator implements TaskDecorator {
+
+    @Override
+    @NonNull
+    public Runnable decorate(@NonNull Runnable runnable) {
+
+        String tenantId = TenantContext.getTenantId();
+        return () -> {
+            try {
+                TenantContext.setTenantId(tenantId);
+                runnable.run();
+            } finally {
+                TenantContext.setTenantId(null);
+            }
+        };
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantContext.java

@@ -0,0 +1,29 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public final class TenantContext {
+
+    private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
+
+
+    public static void setTenantId(String tenantId) {
+
+        log.debug("Setting tenantId to " + tenantId);
+        currentTenant.set(tenantId);
+    }
+
+
+    public static String getTenantId() {
+
+        return currentTenant.get();
+    }
+
+
+    public static void clear() {
+
+        currentTenant.remove();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantInterceptor.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Component;
+import org.springframework.ui.ModelMap;
+import org.springframework.web.context.request.WebRequest;
+import org.springframework.web.context.request.WebRequestInterceptor;
+
+@Component
+public class TenantInterceptor implements WebRequestInterceptor {
+
+    public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
+
+
+    @Override
+    public void preHandle(WebRequest request) {
+
+        if (request.getHeader(TENANT_HEADER_NAME) != null) {
+            TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
+        }
+    }
+
+
+    @Override
+    public void postHandle(WebRequest request, ModelMap model) {
+
+        TenantContext.clear();
+    }
+
+
+    @Override
+    public void afterCompletion(WebRequest request, Exception ex) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -12,8 +12,8 @@ import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -10,8 +10,8 @@ import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 import java.util.HashSet;
 import java.util.Set;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 
 import lombok.*;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/FindEntitiesResult.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PageEntities.java

@@ -6,7 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -18,11 +18,11 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class TenantDictionary {
+
+    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
+    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -1,25 +1,46 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import java.awt.Color;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.SerializationUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary;
 
 import feign.FeignException;
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.SerializationUtils;
-import org.springframework.stereotype.Service;
-
-import java.awt.*;
-import java.util.List;
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -28,22 +49,40 @@ public class DictionaryService {
     private static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
-    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
-    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
+    @Value("${multitenancy.dictionary-cache.maximumSize:100}")
+    private Long maximumSize;
+
+    @Value("${multitenancy.dictionary-cache.expireAfterAccess:3}")
+    private Integer expireAfterAccess;
+
+    private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
 
 
+    @PostConstruct
+    protected void createCache() {
+
+        tenantDictionaryCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(expireAfterAccess, TimeUnit.DAYS).build(new CacheLoader<>() {
+            public TenantDictionary load(String key) {
+
+                return new TenantDictionary();
+            }
+        });
+    }
+
+
+    @SneakyThrows
     @Timed("redactmanager_updateDictionary")
     public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
 
         log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
         long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
-        var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
+        var dossierTemplateDictionary = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
         if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
         }
 
         long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
-        var dossierDictionary = dictionariesByDossier.get(dossierId);
+        var dossierDictionary = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
         if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
         }
@@ -52,13 +91,14 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     @Timed("redactmanager_getDictionaryIncrements")
     public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
 
         DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
 
         Set<DictionaryIncrementValue> newValues = new HashSet<>();
-        List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
+        List<DictionaryModel> dictionaryModels = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getDictionary();
         dictionaryModels.forEach(dictionaryModel -> {
             dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                 if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
@@ -77,8 +117,8 @@ public class DictionaryService {
             });
         });
 
-        if (dictionariesByDossier.containsKey(dossierId)) {
-            dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
+        if (tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId)) {
+            dictionaryModels = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId).getDictionary();
             dictionaryModels.forEach(dictionaryModel -> {
                 dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                     if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
@@ -102,6 +142,7 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {
 
         try {
@@ -115,10 +156,20 @@ public class DictionaryService {
 
                     Optional<DictionaryModel> oldModel;
                     if (dossierId == null) {
-                        var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
+                        DictionaryRepresentation representation = null;
+                        try {
+                            representation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
+                        } catch (ExecutionException e) {
+                            throw new RuntimeException("Failed to load Dictionary cache for tenant: " + TenantContext.getTenantId());
+                        }
                         oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     } else {
-                        var representation = dictionariesByDossier.get(dossierId);
+                        DictionaryRepresentation representation = null;
+                        try {
+                            representation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
+                        } catch (ExecutionException e) {
+                            throw new RuntimeException("Failed to load Dictionary cache for tenant: " + TenantContext.getTenantId());
+                        }
                         oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     }
 
@@ -178,9 +229,9 @@ public class DictionaryService {
                 dictionaryRepresentation.setDictionary(dictionary);
 
                 if (dossierId == null) {
-                    dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
+                    tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation);
                 } else {
-                    dictionariesByDossier.put(dossierId, dictionaryRepresentation);
+                    tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation);
                 }
             }
         } catch (FeignException e) {
@@ -219,19 +270,21 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     public float[] getColor(String type, String dossierTemplateId) {
 
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getLocalAccessMap().get(type);
         if (model != null) {
             return model.getColor();
         }
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getDefaultColor();
     }
 
 
+    @SneakyThrows
     public boolean isHint(String type, String dossierTemplateId) {
 
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getLocalAccessMap().get(type);
         if (model != null) {
             return model.isHint();
         }
@@ -239,20 +292,21 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     @Timed("redactmanager_getDeepCopyDictionary")
     public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
 
         List<DictionaryModel> copy = new ArrayList<>();
 
-        var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
+        var dossierTemplateRepresentation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
         dossierTemplateRepresentation.getDictionary().forEach(dm -> {
             copy.add(SerializationUtils.clone(dm));
         });
 
         //TODO merge dictionaries if they have same names
         long dossierDictionaryVersion = -1;
-        if (dictionariesByDossier.containsKey(dossierId)) {
-            var dossierRepresentation = dictionariesByDossier.get(dossierId);
+        if (tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId)) {
+            var dossierRepresentation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
             dossierRepresentation.getDictionary().forEach(dm -> {
                 copy.add(SerializationUtils.clone(dm));
             });
@@ -264,9 +318,10 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     public float[] getNotRedactedColor(String dossierTemplateId) {
 
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getNotRedactedColor();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java

@@ -5,11 +5,11 @@ import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.ImportedRedaction;
-import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -51,8 +51,8 @@ public class ImportedRedactionService {
                                                                              List<RedactionLogEntry> redactionLogEntries,
                                                                              ImportedRedactions importedRedactions) {
 
-        for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
-            for (ImportedRedaction importedRedaction : importedRedactionsValues) {
+        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
+            for (var importedRedaction : importedRedactionsValues) {
                 RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
                         .id(importedRedaction.getId())
                         .type(IMPORTED_REDACTION_TYPE)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -7,12 +7,11 @@ import java.util.Set;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -23,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
@@ -125,7 +125,7 @@ public class ManualRedactionSurroundingTextService {
     }
 
 
-    public boolean intersects(Rectangle manualPosition, com.iqser.red.service.redaction.v1.model.Rectangle textPositionRectangle) {
+    public boolean intersects(Rectangle manualPosition, com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle textPositionRectangle) {
 
         return textPositionRectangle.getTopLeft().getX() + textPositionRectangle.getWidth() > manualPosition.getTopLeftX() && textPositionRectangle.getTopLeft()
                 .getY() + textPositionRectangle.getHeight() > manualPosition.getTopLeftY() && textPositionRectangle.getTopLeft()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -9,9 +9,9 @@ import java.util.Set;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -1,545 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import java.awt.Color;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.Change;
-import com.iqser.red.service.redaction.v1.model.ChangeType;
-import com.iqser.red.service.redaction.v1.model.ManualChange;
-import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class RedactionLogMergeService {
-
-    private static final String DELETED_TYPE_COLOR = "#9398a0";
-    private final SectionTextService sectionTextService;
-
-    private final RedactionStorageService redactionStorageService;
-
-
-    @Timed("redactmanager_getMergedRedactionLog")
-    public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
-
-        log.debug("Requested preview for: {}", redactionRequest);
-
-        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-        if (redactionLog == null) {
-            throw new NotFoundException("RedactionLog not present");
-        }
-
-        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
-        if (sectionGrid.getSections().isEmpty()) {
-
-            log.info("SectionGrid does not have headlines set. Computing headlines now!");
-            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-            // enhance section grid with headline data
-            for (var sectionText : text.getSectionTexts()) {
-                sectionGrid.getSections()
-                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
-                                sectionText.getHeadline(),
-                                sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
-                                sectionText.getSectionAreas()));
-            }
-            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
-        }
-
-        log.debug("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        var merged = mergeRedactionLogData(redactionLog,
-                sectionGrid,
-                redactionRequest.getManualRedactions(),
-                redactionRequest.getExcludedPages(),
-                redactionRequest.getTypes(),
-                redactionRequest.getColors());
-
-        merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
-
-        return merged;
-    }
-
-
-    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog,
-                                               SectionGrid sectionGrid,
-                                               ManualRedactions manualRedactions,
-                                               Set<Integer> excludedPages,
-                                               List<Type> types,
-                                               Colors colors) {
-
-        var skippedImportedRedactions = new HashSet<>();
-        log.info("Merging Redaction log with manual redactions");
-        if (manualRedactions != null) {
-
-            var manualRedactionLogEntries = addManualAddEntries(sectionGrid,
-                    manualRedactions.getEntriesToAdd(),
-                    manualRedactions.getComments(),
-                    colors,
-                    types,
-                    redactionLog.getAnalysisNumber());
-
-            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
-
-            var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
-
-            for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-
-                processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())).collect(Collectors.toList()), entry, types, colors);
-
-                if (entry.isImported() && !entry.isRedacted()) {
-                    skippedImportedRedactions.add(entry.getId());
-                }
-
-                entry.setComments(convert(manualRedactions.getComments().get(entry.getId())));
-
-                if (excludedPages != null && !excludedPages.isEmpty()) {
-                    entry.getPositions().forEach(pos -> {
-                        if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
-                            entry.setExcluded(true);
-                        }
-                    });
-                }
-            }
-
-        }
-
-        Set<String> processedIds = new HashSet<>();
-        redactionLog.getRedactionLogEntry().removeIf(entry -> {
-
-            if (entry.getImportedRedactionIntersections() != null) {
-                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
-                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
-                        .equals("ocr")))) {
-                    return true;
-                }
-            }
-            if (processedIds.contains(entry.getId())) {
-                log.info("Duplicate annotation found with id {}", entry.getId());
-                return true;
-            }
-            processedIds.add(entry.getId());
-            return false;
-        });
-        return redactionLog;
-    }
-
-
-    private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
-
-        List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
-
-        manualRedactions.getImageRecategorization().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getIdsToRemove().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getForceRedactions().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getLegalBasisChanges().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getResizeRedactions().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        Collections.sort(manualRedactionWrappers);
-
-        return manualRedactionWrappers;
-    }
-
-
-    private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors) {
-
-        manualRedactionWrappers.forEach(mrw -> {
-
-            if (mrw.getItem() instanceof ManualImageRecategorization) {
-                var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
-                String manualOverrideReason = null;
-                if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-
-                    redactionLogEntry.setType(imageRecategorization.getType());
-                    redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
-
-                    if (isHint(types, imageRecategorization.getType())) {
-                        redactionLogEntry.setRedacted(false);
-                        redactionLogEntry.setHint(true);
-                    } else {
-                        redactionLogEntry.setHint(false);
-                    }
-
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
-                } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                redactionLogEntry.getManualChanges()
-                        .add(ManualChange.from(imageRecategorization)
-                                .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                                .withChange("type", imageRecategorization.getType()));
-            }
-
-            if (mrw.getItem() instanceof IdRemoval) {
-                var manualRemoval = (IdRemoval) mrw.getItem();
-                if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
-                    log.debug("Skipping merge for dictionary-modifying entry");
-                } else {
-                    String manualOverrideReason = null;
-                    if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
-                        redactionLogEntry.setRedacted(false);
-                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), true, types));
-                        redactionLogEntry.setHint(false);
-                    } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    }
-
-                    if (manualOverrideReason != null) {
-                        redactionLogEntry.setReason(manualOverrideReason);
-                    }
-                }
-
-                redactionLogEntry.getManualChanges()
-                        .add(ManualChange.from(manualRemoval)
-                                .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
-            }
-
-            if (mrw.getItem() instanceof ManualForceRedaction) {
-                var manualForceRedact = (ManualForceRedaction) mrw.getItem();
-                String manualOverrideReason = null;
-                var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
-                if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    // Forcing a skipped hint should result in a hint
-                    if (dictionaryIsHint) {
-                        redactionLogEntry.setHint(true);
-                    } else {
-                        redactionLogEntry.setRedacted(true);
-                    }
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-                } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                var manualChange = ManualChange.from(manualForceRedact)
-                        .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
-
-                redactionLogEntry.getManualChanges().add(manualChange);
-
-            }
-
-            if (mrw.getItem() instanceof ManualLegalBasisChange) {
-                var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
-                String manualOverrideReason = null;
-                if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
-                    redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
-                    redactionLogEntry.setRedacted(true);
-                    if (manualLegalBasisChange.getSection() != null) {
-                        redactionLogEntry.setSection(manualLegalBasisChange.getSection());
-                    }
-                    if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                        redactionLogEntry.setValue(manualLegalBasisChange.getValue());
-                    }
-                } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
-                manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
-                if (manualLegalBasisChange.getSection() != null) {
-                    manualChange.withChange("section", manualLegalBasisChange.getSection());
-                }
-                if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                    manualChange.withChange("value", manualLegalBasisChange.getValue());
-                }
-                redactionLogEntry.getManualChanges().add(manualChange);
-            }
-
-            if (mrw.getItem() instanceof ManualResizeRedaction) {
-                var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
-                String manualOverrideReason = null;
-                if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-                    if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
-                        redactionLogEntry.setValue(manualResizeRedact.getValue());
-                    }
-                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-                    }
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
-                } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-
-                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-                    }
-                }
-
-                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.getManualChanges()
-                        .add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
-            }
-
-        });
-
-    }
-
-
-    private String mergeReasonIfNecessary(String currentReason, String addition) {
-
-        if (currentReason != null) {
-            if (!currentReason.contains(addition)) {
-                return currentReason + addition;
-            }
-            return currentReason;
-        } else {
-            return "";
-        }
-    }
-
-
-    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
-                                                       Set<ManualRedactionEntry> manualAdds,
-                                                       Map<String, List<Comment>> comments,
-                                                       Colors colors,
-                                                       List<Type> types,
-                                                       int analysisNumber) {
-
-        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
-
-        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
-
-            if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types, analysisNumber);
-                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
-                redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId())));
-                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
-                redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
-
-                sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
-
-                redactionLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    private List<RedactionLogComment> convert(List<Comment> comments) {
-
-        return comments == null ? null : comments.stream()
-                .map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
-                .collect(Collectors.toList());
-    }
-
-
-    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions) {
-
-        return positions.stream()
-                .map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
-                .collect(Collectors.toList());
-    }
-
-
-    @SuppressWarnings("PMD.UselessParentheses")
-    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, Colors colors, List<Type> types, int analysisNumber) {
-
-        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
-
-        var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
-        List<ManualChange> changeList = new ArrayList<>();
-        changeList.add(change);
-
-        return RedactionLogEntry.builder()
-                .id(id)
-                .color(getColorForManualAdd(manualRedactionEntry.getType(), colors, manualRedactionEntry.getStatus(), types))
-                .reason(manualRedactionEntry.getReason())
-                .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
-                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
-                .legalBasis(manualRedactionEntry.getLegalBasis())
-                .value(manualRedactionEntry.getValue())
-                .sourceId(manualRedactionEntry.getSourceId())
-                .section(manualRedactionEntry.getSection())
-                .type(manualRedactionEntry.getType())
-                .redacted(true)
-                .isHint(false)
-                .sectionNumber(-1)
-                .rectangle(manualRedactionEntry.isRectangle())
-                .manualChanges(changeList)
-                .changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
-                .build();
-    }
-
-
-    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped, List<Type> types) {
-
-        if (requested) {
-            return convertColor(colors.getRequestRemoveColor());
-        }
-        if (skipped || !isRedaction && !isHint(types, type)) {
-            return convertColor(colors.getSkippedColor());
-        }
-        return getColor(types, type);
-    }
-
-
-    private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
-
-        if (status.equals(AnnotationStatus.REQUESTED)) {
-            return convertColor(colors.getRequestAddColor());
-        } else if (status.equals(AnnotationStatus.DECLINED)) {
-            return convertColor(colors.getSkippedColor());
-        }
-        return getColor(types, type);
-    }
-
-
-    private float[] getColor(List<Type> types, String type) {
-
-        var matchingTypes = getMatchingTypes(types, type);
-        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
-        if (foundAndNotDeletedType.isPresent()) {
-            return convertColor(foundAndNotDeletedType.get().getHexColor());
-        }
-        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
-        return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor(DELETED_TYPE_COLOR));
-    }
-
-
-    boolean isHint(List<Type> types, String type) {
-
-        var matchingTypes = getMatchingTypes(types, type);
-        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
-        if (foundAndNotDeletedType.isPresent()) {
-            return foundAndNotDeletedType.get().isHint();
-        }
-        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
-        return firstDeletedType.map(Type::isHint).orElse(false);
-    }
-
-
-    private float[] convertColor(String hex) {
-
-        Color color = Color.decode(hex);
-        return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
-    }
-
-
-    private List<Type> getMatchingTypes(List<Type> types, String type) {
-
-        return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
-    }
-
-
-    private boolean isDeletedType(Type type) {
-
-        return type.getSoftDeletedTime() != null;
-    }
-
-
-    @Data
-    @AllArgsConstructor
-    private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
-
-        private String id;
-        private OffsetDateTime date;
-        private Object item;
-
-
-        @Override
-        public int compareTo(ManualRedactionWrapper o) {
-
-            return this.date.compareTo(o.date);
-        }
-
-    }
-
-}
-

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.model.CellRectangle;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.SectionRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -11,8 +11,8 @@ import java.util.stream.Collectors;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java

@@ -1,41 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Service;
-
-@Slf4j
-@Service
-public class SectionTextService {
-
-    public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) {
-
-        if (redactionLogEntry.getSection() != null) {
-            // set by UI
-            return;
-        }
-
-        if (sectionGrid != null) {
-            var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null;
-
-            if (firstPosition != null) {
-
-                for (var section : sectionGrid.getSections()) {
-                    if (section.getPages().contains(firstPosition.getPage())) {
-                        for (var sectionArea : section.getSectionAreas()) {
-                            if (sectionArea.contains(firstPosition)) {
-                                redactionLogEntry.setSection(section.getHeadline());
-                                redactionLogEntry.setSectionNumber(section.getSectionNumber());
-                                return;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -8,23 +8,23 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@@ -53,10 +53,8 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
-import io.micrometer.core.instrument.FunctionTimer;
-import io.micrometer.core.instrument.MeterRegistry;
 import lombok.AccessLevel;
-import lombok.Getter;
+import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
@@ -64,8 +62,11 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Service
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@RequiredArgsConstructor
 public class AnalyzeService {
 
+    private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
+
     DictionaryService dictionaryService;
     DroolsExecutionService droolsExecutionService;
     EntityRedactionService entityRedactionService;
@@ -80,50 +81,8 @@ public class AnalyzeService {
     ImageService imageService;
     ImportedRedactionService importedRedactionService;
     SectionFinder sectionFinder;
-    PagewiseCounter analyzeCounter = new PagewiseCounter();
 
-
-    @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
-    public AnalyzeService(DictionaryService dictionaryService,
-                          DroolsExecutionService droolsExecutionService,
-                          EntityRedactionService entityRedactionService,
-                          RedactionLogCreatorService redactionLogCreatorService,
-                          RedactionStorageService redactionStorageService,
-                          PdfSegmentationService pdfSegmentationService,
-                          RedactionChangeLogService redactionChangeLogService,
-                          LegalBasisClient legalBasisClient,
-                          RedactionServiceSettings redactionServiceSettings,
-                          SectionTextBuilderService sectionTextBuilderService,
-                          SectionGridCreatorService sectionGridCreatorService,
-                          ImageService imageService,
-                          ImportedRedactionService importedRedactionService,
-                          SectionFinder sectionFinder,
-                          MeterRegistry meterRegistry) {
-
-        this.dictionaryService = dictionaryService;
-        this.droolsExecutionService = droolsExecutionService;
-        this.entityRedactionService = entityRedactionService;
-        this.redactionLogCreatorService = redactionLogCreatorService;
-        this.redactionStorageService = redactionStorageService;
-        this.pdfSegmentationService = pdfSegmentationService;
-        this.redactionChangeLogService = redactionChangeLogService;
-        this.legalBasisClient = legalBasisClient;
-        this.redactionServiceSettings = redactionServiceSettings;
-        this.sectionTextBuilderService = sectionTextBuilderService;
-        this.sectionGridCreatorService = sectionGridCreatorService;
-        this.imageService = imageService;
-        this.importedRedactionService = importedRedactionService;
-        this.sectionFinder = sectionFinder;
-
-        // This is just here as an example of how to do a timer with a counter.
-        // For a full implementation this should be moved to a common helper class.
-        FunctionTimer.builder("redactmanager_analyze.pagewise", //
-                        analyzeCounter, //
-                        PagewiseCounter::getTotalPageCount, //
-                        PagewiseCounter::getTotalTimeMillis, //
-                        TimeUnit.MILLISECONDS) //
-                .register(meterRegistry);
-    }
+    FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
 
     @Timed("redactmanager_analyzeDocumentStructure")
@@ -296,7 +255,7 @@ public class AnalyzeService {
 
         long duration = System.currentTimeMillis() - startTime;
 
-        analyzeCounter.increase(text.getNumberOfPages(), duration);
+        redactmanagerAnalyzePagewiseValues.increase(text.getNumberOfPages(), duration);
 
         return AnalyzeResult.builder()
                 .dossierId(analyzeRequest.getDossierId())
@@ -347,21 +306,4 @@ public class AnalyzeService {
         return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
     }
 
-
-    @FieldDefaults(level = AccessLevel.PRIVATE)
-    @Getter
-    private static final class PagewiseCounter {
-
-        long totalPageCount;
-        long totalTimeMillis;
-
-
-        public void increase(long pageCount, long durationMillis) {
-
-            totalPageCount += pageCount;
-            totalTimeMillis += durationMillis;
-        }
-
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java

@@ -9,16 +9,16 @@ import java.util.stream.Stream;
 
 import org.springframework.stereotype.Component;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java

@@ -8,8 +8,8 @@ import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java

@@ -12,11 +12,11 @@ import java.util.stream.Stream;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -11,8 +11,8 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/FindEntityDetails.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java

@@ -9,7 +9,7 @@ import java.util.Map;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TableService.java

@@ -9,7 +9,7 @@ import java.util.Map;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
 import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableCells;
 import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableServiceResponse;
@@ -60,4 +60,4 @@ public class TableService {
         return pdfTableCells;
     }
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -4,13 +4,14 @@ import java.io.InputStream;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 
@@ -31,14 +32,14 @@ public class RedactionStorageService {
     @SneakyThrows
     public InputStream getStoredObject(String storageId) {
 
-        return storageService.getObject(storageId).getInputStream();
+        return storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream();
     }
 
 
     @SneakyThrows
     public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) {
 
-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
+        storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
     }
 
 
@@ -46,7 +47,7 @@ public class RedactionStorageService {
     @Timed("redactmanager_storeObject")
     public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
 
-        storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
+        storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
     }
 
 
@@ -54,7 +55,7 @@ public class RedactionStorageService {
     public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Imported redactions not available.");
             return null;
@@ -66,7 +67,7 @@ public class RedactionStorageService {
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
@@ -79,7 +80,7 @@ public class RedactionStorageService {
     public Text getText(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
@@ -91,7 +92,7 @@ public class RedactionStorageService {
     public NerEntities getNerEntities(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("NER Entities are not available.");
         }
@@ -102,7 +103,7 @@ public class RedactionStorageService {
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("Section Grid is not available.");
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;
 
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 

redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml

@@ -1,11 +1,17 @@
 server:
   port: 8083
 
-configuration-service.url: "http://localhost:8081"
-file-management-service.url: "http://localhost:8085"
+persistence-service.url: "http://localhost:8085"
 
 storage:
   bucket-name: 'redaction'
   endpoint: 'http://localhost:9000'
   key: minioadmin
   secret: minioadmin
+
+
+redaction-service:
+  enableImageClassification: false
+  cvTableParsingEnabled: false
+  nerServiceEnabled: false
+  priorityMode: false

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -38,8 +38,4 @@ management:
 
 
 storage:
-  signer-type: 'AWSS3V4SignerType'
-  bucket-name: 'redaction'
-  region: 'us-east-1'
-  endpoint: 'https://s3.amazonaws.com'
   backend: 's3'

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java

@@ -1,503 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import static org.mockito.Mockito.when;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.stubbing.Answer;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
-@ExtendWith(SpringExtension.class)
-@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(AbstractTestWithDictionaries.TestConfiguration.class)
-public class AbstractTestWithDictionaries {
-
-    protected static final String RULES = loadFromClassPath("drools/rules.drl");
-    protected static final String RULES_PATH = "drools/rules.drl";
-    protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
-    private static final String VERTEBRATE = "vertebrate";
-    private static final String ADDRESS = "CBI_address";
-    private static final String AUTHOR = "CBI_author";
-    private static final String SPONSOR = "CBI_sponsor";
-    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
-    private static final String REDACTION_INDICATOR = "redaction_indicator";
-    private static final String HINT_ONLY = "hint_only";
-    private static final String MUST_REDACT = "must_redact";
-    private static final String PUBLISHED_INFORMATION = "published_information";
-    private static final String TEST_METHOD = "test_method";
-    private static final String PURITY = "purity";
-    private static final String IMAGE = "image";
-    private static final String LOGO = "logo";
-    private static final String SIGNATURE = "signature";
-    private static final String FORMULA = "formula";
-    private static final String OCR = "ocr";
-    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
-    private static final String IMPORTED_REDACTION = "imported_redaction";
-    private static final String PII = "PII";
-    private static final String ROTATE_SIMPLE = "RotateSimple";
-
-    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
-    protected final static String TEST_DOSSIER_ID = "123";
-    protected final static String TEST_FILE_ID = "123";
-
-    @Autowired
-    protected StorageService storageService;
-
-    @MockBean
-    protected DictionaryClient dictionaryClient;
-
-    @MockBean
-    protected RulesClient rulesClient;
-
-    private final Map<String, List<String>> dictionary = new HashMap<>();
-    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
-    private final Map<String, List<String>> falsePositive = new HashMap<>();
-    private final Map<String, List<String>> falseRecommendation = new HashMap<>();
-    private final Map<String, String> typeColorMap = new HashMap<>();
-    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
-    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
-    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
-    private final Map<String, Integer> rankTypeMap = new HashMap<>();
-    private final Colors colors = new Colors();
-    private final Map<String, Long> reanalysisVersions = new HashMap<>();
-    private final Set<String> deleted = new HashSet<>();
-
-
-    @BeforeEach
-    public void stubClients() {
-
-        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
-
-        loadDictionaryForTest();
-        loadTypeForTest();
-        loadNerForTest();
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
-
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS)
-                .dossierTemplateId(TEST_DOSSIER_ID)
-                .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
-                .build()));
-
-        mockDictionaryCalls(null);
-        mockDictionaryCalls(0L);
-
-        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
-    }
-
-
-    private static String loadFromClassPath(String path) {
-
-        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
-        }
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
-            StringBuilder sb = new StringBuilder();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
-        }
-    }
-
-
-    private void mockDictionaryCalls(Long version) {
-
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
-                false));
-        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
-        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                NO_REDACTION_INDICATOR,
-                false));
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                REDACTION_INDICATOR,
-                false));
-        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
-                false));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                PUBLISHED_INFORMATION,
-                false));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
-                false));
-        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
-        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
-        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
-        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
-        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
-        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
-        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
-        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
-                false));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                DOSSIER_REDACTIONS,
-                true));
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                IMPORTED_REDACTION,
-                true));
-
-    }
-
-
-    private String cleanDictionaryEntry(String entry) {
-
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-
-
-    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
-
-        if (entries == null) {
-            entries = Collections.emptyList();
-        }
-
-        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
-        entries.forEach(entry -> {
-            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
-        });
-        return dictionaryEntries;
-    }
-
-
-    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
-
-        return Type.builder()
-                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .hexColor(typeColorMap.get(type))
-                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
-                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
-                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
-                .isHint(hintTypeMap.get(type))
-                .isCaseInsensitive(caseInSensitiveMap.get(type))
-                .isRecommendation(recommendationTypeMap.get(type))
-                .rank(rankTypeMap.get(type))
-                .build();
-    }
-
-
-    private void loadTypeForTest() {
-
-        typeColorMap.put(VERTEBRATE, "#ff85f7");
-        typeColorMap.put(ADDRESS, "#ffe187");
-        typeColorMap.put(AUTHOR, "#ffe187");
-        typeColorMap.put(SPONSOR, "#85ebff");
-        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
-        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
-        typeColorMap.put(HINT_ONLY, "#abc0c4");
-        typeColorMap.put(MUST_REDACT, "#fab4c0");
-        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
-        typeColorMap.put(TEST_METHOD, "#91fae8");
-        typeColorMap.put(PII, "#66ccff");
-        typeColorMap.put(PURITY, "#ffe187");
-        typeColorMap.put(IMAGE, "#fcc5fb");
-        typeColorMap.put(OCR, "#fcc5fb");
-        typeColorMap.put(LOGO, "#ffe187");
-        typeColorMap.put(FORMULA, "#ffe187");
-        typeColorMap.put(SIGNATURE, "#ffe187");
-        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
-        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
-
-        hintTypeMap.put(VERTEBRATE, true);
-        hintTypeMap.put(ADDRESS, false);
-        hintTypeMap.put(AUTHOR, false);
-        hintTypeMap.put(SPONSOR, false);
-        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
-        hintTypeMap.put(REDACTION_INDICATOR, true);
-        hintTypeMap.put(HINT_ONLY, true);
-        hintTypeMap.put(MUST_REDACT, true);
-        hintTypeMap.put(PUBLISHED_INFORMATION, true);
-        hintTypeMap.put(TEST_METHOD, true);
-        hintTypeMap.put(PII, false);
-        hintTypeMap.put(PURITY, false);
-        hintTypeMap.put(IMAGE, true);
-        hintTypeMap.put(OCR, true);
-        hintTypeMap.put(FORMULA, false);
-        hintTypeMap.put(LOGO, false);
-        hintTypeMap.put(SIGNATURE, false);
-        hintTypeMap.put(DOSSIER_REDACTIONS, false);
-        hintTypeMap.put(IMPORTED_REDACTION, false);
-        hintTypeMap.put(ROTATE_SIMPLE, false);
-
-        caseInSensitiveMap.put(VERTEBRATE, true);
-        caseInSensitiveMap.put(ADDRESS, false);
-        caseInSensitiveMap.put(AUTHOR, false);
-        caseInSensitiveMap.put(SPONSOR, false);
-        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(HINT_ONLY, true);
-        caseInSensitiveMap.put(MUST_REDACT, true);
-        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
-        caseInSensitiveMap.put(TEST_METHOD, false);
-        caseInSensitiveMap.put(PII, false);
-        caseInSensitiveMap.put(PURITY, false);
-        caseInSensitiveMap.put(IMAGE, true);
-        caseInSensitiveMap.put(OCR, true);
-        caseInSensitiveMap.put(SIGNATURE, true);
-        caseInSensitiveMap.put(LOGO, true);
-        caseInSensitiveMap.put(FORMULA, true);
-        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
-        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
-        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
-
-        recommendationTypeMap.put(VERTEBRATE, false);
-        recommendationTypeMap.put(ADDRESS, false);
-        recommendationTypeMap.put(AUTHOR, false);
-        recommendationTypeMap.put(SPONSOR, false);
-        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(HINT_ONLY, false);
-        recommendationTypeMap.put(MUST_REDACT, false);
-        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
-        recommendationTypeMap.put(TEST_METHOD, false);
-        recommendationTypeMap.put(PII, false);
-        recommendationTypeMap.put(PURITY, false);
-        recommendationTypeMap.put(IMAGE, false);
-        recommendationTypeMap.put(OCR, false);
-        recommendationTypeMap.put(FORMULA, false);
-        recommendationTypeMap.put(SIGNATURE, false);
-        recommendationTypeMap.put(LOGO, false);
-        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
-        recommendationTypeMap.put(IMPORTED_REDACTION, false);
-        recommendationTypeMap.put(ROTATE_SIMPLE, false);
-
-        rankTypeMap.put(PURITY, 155);
-        rankTypeMap.put(PII, 150);
-        rankTypeMap.put(ADDRESS, 140);
-        rankTypeMap.put(AUTHOR, 130);
-        rankTypeMap.put(SPONSOR, 120);
-        rankTypeMap.put(VERTEBRATE, 110);
-        rankTypeMap.put(MUST_REDACT, 100);
-        rankTypeMap.put(REDACTION_INDICATOR, 90);
-        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
-        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
-        rankTypeMap.put(TEST_METHOD, 60);
-        rankTypeMap.put(HINT_ONLY, 50);
-        rankTypeMap.put(IMAGE, 30);
-        rankTypeMap.put(OCR, 29);
-        rankTypeMap.put(LOGO, 28);
-        rankTypeMap.put(SIGNATURE, 27);
-        rankTypeMap.put(FORMULA, 26);
-        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
-        rankTypeMap.put(IMPORTED_REDACTION, 200);
-        rankTypeMap.put(ROTATE_SIMPLE, 150);
-
-        colors.setSkippedColor("#cccccc");
-        colors.setRequestAddColor("#04b093");
-        colors.setRequestRemoveColor("#04b093");
-    }
-
-
-    @SneakyThrows
-    private void loadNerForTest() {
-
-        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
-    }
-
-
-    private List<Type> getTypeResponse() {
-
-        return typeColorMap.entrySet()
-                .stream()
-                .map(typeColor -> Type.builder()
-                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                        .type(typeColor.getKey())
-                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                        .hexColor(typeColor.getValue())
-                        .isHint(hintTypeMap.get(typeColor.getKey()))
-                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
-                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
-                        .rank(rankTypeMap.get(typeColor.getKey()))
-                        .build())
-
-                .collect(Collectors.toList());
-    }
-
-
-    protected List<File> getPathsRecursively(File path) {
-
-        List<File> result = new ArrayList<>();
-        if (path == null || path.listFiles() == null) {
-            return result;
-        }
-        for (File f : path.listFiles()) {
-            if (f.isFile()) {
-                result.add(f);
-            } else {
-                result.addAll(getPathsRecursively(f));
-            }
-        }
-        return result;
-
-    }
-
-
-    protected void loadOnlyDictionaryForSimpleFile() {
-
-        dictionary.clear();
-        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-    }
-
-
-    private void loadDictionaryForTest() {
-
-        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
-
-        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(String file) {
-
-        return prepareStorage(file, "files/cv_service_empty_response.json");
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
-
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
-        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
-
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
-
-        AnalyzeRequest request = AnalyzeRequest.builder()
-                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .lastProcessed(OffsetDateTime.now())
-                .build();
-
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
-
-        return request;
-    }
-
-
-    @AfterEach
-    public void cleanupStorage() {
-
-        if (this.storageService instanceof FileSystemBackedStorageService) {
-            ((FileSystemBackedStorageService) this.storageService).clearStorage();
-        }
-    }
-
-
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
-    public static class TestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService();
-        }
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisIntegrationTest.java

@@ -1,213 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import javax.servlet.ServletContext;
-
-import org.apache.commons.collections4.SetUtils;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.core.io.ClassPathResource;
-
-import com.google.common.io.Files;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import lombok.Builder;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class AnalysisIntegrationTest extends AbstractTestWithDictionaries {
-
-    @Autowired
-    private AnalyzeService analyzeService;
-
-    @Autowired
-    private ServletContext context;
-
-    @Autowired
-    private RedactionStorageService redactionStorageService;
-
-    @MockBean
-    private RabbitTemplate rabbitTemplate;
-
-    @MockBean
-    private LegalBasisClient legalBasisClient;
-
-
-    @Test
-    public void testAnalysisIntegrationForAllFiles() {
-
-        findAllFilesWithRedactionLogs().forEach((pdfFile, redactionLogFile) -> {
-
-            try {
-                String relativePdfFilePath = new ClassPathResource("").getURI().relativize(pdfFile.toURI()).getPath();
-                log.info("Checking equality for {}", relativePdfFilePath);
-                AnalyzeRequest request = prepareStorage(relativePdfFilePath);
-
-                analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
-                analyzeService.analyze(request);
-
-                RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-                Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
-
-                Path entityReferencePath = Paths.get(new ClassPathResource("entities/").getFile().getAbsolutePath().replace("target/test-classes/", "src/test/resources/"),
-                        relativePdfFilePath.replace("files/", "").replace(".pdf", ".json"));
-
-                new File(entityReferencePath.toString().replace(entityReferencePath.getFileName().toString(), "")).mkdirs();
-
-                try (var out = new FileOutputStream(entityReferencePath.toString())) {
-                    ObjectMapperFactory.create().writeValue(out, references);
-                    log.info("{} Entities have been written for File {}", references.size(), relativePdfFilePath);
-                }
-                //RedactionLog originalRedactionLog = ObjectMapperFactory.create().readValue(redactionLogFile, RedactionLog.class);
-                //assertEntitiesAreEqual(originalRedactionLog, redactionLog);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-
-        });
-    }
-
-
-    // only use this if you must replace all entities with newer versions
-    @Test
-    @Disabled
-    @SneakyThrows
-    public void writeNewEntitiesForAllFiles() {
-
-        ClassPathResource fileStem = new ClassPathResource("files/");
-        List<File> allPdfFiles = getAllPdfFilesAsStream(fileStem).toList();
-        System.out.printf("found %d PDF Files to create Entities for\n", allPdfFiles.size());
-        int count = 0;
-        for (File pdfFile : allPdfFiles) {
-            writeNewEntities(pdfFile);
-            System.out.printf("%d|%d finished writing entities for %s\n", count, allPdfFiles.size(), pdfFile.getName());
-            count++;
-        }
-    }
-
-
-    private static Stream<File> getAllPdfFilesAsStream(ClassPathResource fileStem) throws IOException {
-
-        return Arrays.stream(fileStem.getFile().listFiles())
-                .filter(File::isDirectory)
-                .flatMap(file -> Arrays.stream(file.listFiles()))
-                .filter(file -> Files.getFileExtension(file.getName()).equals("pdf"))
-                .filter(File::isFile)
-                .filter(File::canRead);
-    }
-
-
-    @SneakyThrows
-    private void writeNewEntities(File pdfFile) {
-
-        Path relativePdfFilePath = Paths.get(pdfFile.toString().replace(new ClassPathResource("").getFile().getAbsolutePath(), ""));
-        Path relativeFolderPath = relativePdfFilePath.getParent();
-        Path absoluteEntityFolderPath = Paths.get(context.getContextPath(), "src", "test", "resources", "entities", relativeFolderPath.toString().replace("files/", ""));
-        absoluteEntityFolderPath.toFile().mkdirs();
-        Path absoluteEntityFilePath = absoluteEntityFolderPath.resolve(relativePdfFilePath.getFileName().toString().replace("files/", "").replace(".pdf", ".json"));
-
-        AnalyzeRequest request = prepareStorage(relativePdfFilePath.toString());
-
-        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
-        analyzeService.analyze(request);
-
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
-
-        try (var out = new FileOutputStream(absoluteEntityFilePath.toString())) {
-            ObjectMapperFactory.create().writeValue(out, references);
-        }
-
-    }
-
-
-    private void assertEntitiesAreEqual(RedactionLog originalRedactionLog, RedactionLog redactionLog) {
-
-        Set<EntityReference> originalReferences = originalRedactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
-        Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
-
-        Set<EntityReference> missingReferences = SetUtils.difference(originalReferences, references);
-        Set<EntityReference> additionalReferences = SetUtils.difference(references, originalReferences);
-
-        if (missingReferences.size() == 0) {
-            log.info("All original Entities have been found!");
-        } else {
-            log.warn("{} original References not found {}", missingReferences.size(), missingReferences);
-            assert missingReferences.size() == 0;
-        }
-
-        if (additionalReferences.size() == 0) {
-            log.info("No additional Entities have been found!");
-        } else {
-            log.warn("{} additional Entities found", additionalReferences.size());
-            additionalReferences.forEach(e -> log.warn(e.toString()));
-            assert additionalReferences.size() == 0;
-        }
-    }
-
-
-    private EntityReference toReference(RedactionLogEntry redactionLogEntry) {
-
-        return EntityReference.builder()
-                .positions(redactionLogEntry.getPositions())
-                .isRecommendation(redactionLogEntry.isRecommendation())
-                .isFalsePositive(redactionLogEntry.isFalsePositive())
-                .word(redactionLogEntry.getValue())
-                .isHint(redactionLogEntry.isHint())
-                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
-                .reason(redactionLogEntry.getReason())
-                .build();
-    }
-
-
-    @Builder
-    public record EntityReference(
-            String word, String reason, boolean redacted, boolean isHint, boolean isRecommendation, boolean isFalsePositive, boolean isDictionaryEntry, List<Rectangle> positions) {
-
-    }
-
-
-    @SneakyThrows
-    private Map<File, File> findAllFilesWithRedactionLogs() {
-
-        ClassPathResource fileStem = new ClassPathResource("files/");
-        ClassPathResource redactionStem = new ClassPathResource("RedactionLog/files/");
-
-        Map<File, File> fileToRedactionLogFile = new HashMap<>();
-        List<File> allFiles = getAllPdfFilesAsStream(fileStem).toList();
-
-        for (File file : allFiles) {
-            String relativePath = fileStem.getURI().relativize(file.toURI()).getPath().replace(".pdf", ".json");
-            File redactionLogFile = Paths.get(redactionStem.getFile().getAbsolutePath(), relativePath).toFile();
-            if (redactionLogFile.isFile() && redactionLogFile.canRead()) {
-                fileToRedactionLogFile.put(file, redactionLogFile);
-            }
-        }
-        return fileToRedactionLogFile;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -23,10 +23,11 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -63,6 +64,8 @@ public class DictionaryServiceTest {
     @Test
     public void testDictionaryServiceConsistency() {
 
+        TenantContext.setTenantId("redaction");
+
         when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
         when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId",
                 "#cccccc",
@@ -75,6 +78,7 @@ public class DictionaryServiceTest {
                 "#cccccc",
                 "#cccccc",
                 "#cccccc",
+                "#cccccc",
                 "#cccccc"));
 
         var type1 = new Type();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -33,7 +33,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @SneakyThrows
     @Override
-    public InputStreamResource getObject(String objectId) {
+    public InputStreamResource getObject(String tenantId, String objectId) {
 
         var res = dataMap.get(objectId);
         if (res == null) {
@@ -45,28 +45,22 @@ public class FileSystemBackedStorageService implements StorageService {
 
 
     @Override
-    public void deleteObject(String objectId) {
+    public void deleteObject(String tenantId, String objectId) {
 
         dataMap.remove(objectId);
     }
 
 
     @Override
-    public boolean objectExists(String objectId) {
+    public boolean objectExists(String tenantId, String objectId) {
 
         return dataMap.containsKey(objectId);
     }
 
 
-    @Override
-    public void init() {
-
-    }
-
-
     @Override
     @SneakyThrows
-    public <T> void storeJSONObject(String objectId, T any) {
+    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
 
         File tempFile = FileUtils.createTempFile("test", ".tmp");
         getMapper().writeValue(new FileOutputStream(tempFile), any);
@@ -82,7 +76,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @Override
     @SneakyThrows
-    public <T> T readJSONObject(String objectId, Class<T> clazz) {
+    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
 
         if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
             throw new StorageObjectDoesNotExist("Stored object not found");
@@ -105,7 +99,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @Override
     @SneakyThrows
-    public void storeObject(String objectId, InputStream stream) {
+    public void storeObject(String tenantId, String objectId, InputStream stream) {
 
         File tempFile = FileUtils.createTempFile("test", ".tmp");
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -42,23 +42,22 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
-
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*;
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.ChangeType;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
@@ -241,6 +240,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
 
@@ -362,8 +362,8 @@ public class HeadlinesGoldStandardIntegrationTest {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
 
         return request;
     }
@@ -392,7 +392,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     private void loadNerForTest() {
 
         ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -51,27 +51,25 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
@@ -81,6 +79,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
@@ -214,6 +213,8 @@ public class RedactionIntegrationTest {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
+
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
 
@@ -424,12 +425,7 @@ public class RedactionIntegrationTest {
         request.setManualRedactions(manualRedactions);
         analyzeService.reanalyze(request);
 
-        var mergedRedactionLog = redactionController.getRedactionLog(RedactionRequest.builder()
-                .manualRedactions(manualRedactions)
-                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
+        var mergedRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
         assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
@@ -648,7 +644,7 @@ public class RedactionIntegrationTest {
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
         ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
 
         long start = System.currentTimeMillis();
         AnalyzeRequest request = prepareStorage(fileName);
@@ -1106,109 +1102,8 @@ public class RedactionIntegrationTest {
     }
 
 
-    @Test
-    public void classificationTest() throws IOException {
-
-        System.out.println("classificationTest");
-
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
-
-        RedactionRequest redactionRequest = RedactionRequest.builder()
-                .dossierId(request.getDossierId())
-                .fileId(request.getFileId())
-                .dossierTemplateId(request.getDossierTemplateId())
-                .build();
-
-        RedactionResult result = redactionController.classify(redactionRequest);
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
-            fileOutputStream.write(result.getDocument());
-        }
-    }
 
 
-    @Test
-    public void classificationTestWithCvTableService() throws IOException {
-
-        System.out.println("classificationTest");
-
-        String tableServiceResponseFile = "files/cv_table_response_VV-511309.json";
-        AnalyzeRequest request = prepareStorage("files/new/VV-511309_OCR.pdf", tableServiceResponseFile);
-
-        RedactionRequest redactionRequest = RedactionRequest.builder()
-                .dossierId(request.getDossierId())
-                .fileId(request.getFileId())
-                .dossierTemplateId(request.getDossierTemplateId())
-                .build();
-
-        RedactionResult result = redactionController.classify(redactionRequest);
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
-            fileOutputStream.write(result.getDocument());
-        }
-    }
-
-
-    @Test
-    public void sectionsTest() throws IOException {
-
-        System.out.println("sectionsTest");
-
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/Single Table.pdf");
-
-        RedactionRequest redactionRequest = RedactionRequest.builder()
-                .dossierId(request.getDossierId())
-                .fileId(request.getFileId())
-                .dossierTemplateId(request.getDossierTemplateId())
-                .build();
-
-        RedactionResult result = redactionController.sections(redactionRequest);
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Sections.pdf")) {
-            fileOutputStream.write(result.getDocument());
-        }
-    }
-
-
-    @Test
-    public void htmlTablesTest() throws IOException {
-
-        System.out.println("htmlTablesTest");
-        AnalyzeRequest request = prepareStorage("files/new/table-with-merged-cells.pdf");
-
-        RedactionRequest redactionRequest = RedactionRequest.builder()
-                .dossierId(request.getDossierId())
-                .fileId(request.getFileId())
-                .dossierTemplateId(request.getDossierTemplateId())
-                .build();
-
-        RedactionResult result = redactionController.htmlTables(redactionRequest);
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) {
-            fileOutputStream.write(result.getDocument());
-        }
-    }
-
-
-    @Test
-    public void htmlTableRotationTest() throws IOException {
-
-        System.out.println("htmlTableRotationTest");
-
-        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
-
-        RedactionRequest redactionRequest = RedactionRequest.builder()
-                .dossierId(request.getDossierId())
-                .fileId(request.getFileId())
-                .dossierTemplateId(request.getDossierTemplateId())
-                .build();
-
-        RedactionResult result = redactionController.htmlTables(redactionRequest);
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) {
-            fileOutputStream.write(result.getDocument());
-        }
-    }
 
 
     @Test
@@ -1638,7 +1533,7 @@ public class RedactionIntegrationTest {
     private void loadNerForTest() {
 
         ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
     }
 
 
@@ -1704,7 +1599,7 @@ public class RedactionIntegrationTest {
         ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
 
         AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
                 importedRedactions.getInputStream());
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
@@ -1775,8 +1670,8 @@ public class RedactionIntegrationTest {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
 
         return request;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -52,30 +52,27 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*;
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.Change;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.ManualChange;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
@@ -224,8 +221,6 @@ public class RulesTest {
     @Autowired
     private RedactionController redactionController;
     @Autowired
-    private AnnotationService annotationService;
-    @Autowired
     private AnalyzeService analyzeService;
     @Autowired
     private ObjectMapper objectMapper;
@@ -250,6 +245,7 @@ public class RulesTest {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
         objectMapper.registerModule(new JavaTimeModule());
         objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
@@ -624,7 +620,7 @@ public class RulesTest {
     private void loadNerForTest() {
 
         ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
     }
 
 
@@ -647,9 +643,9 @@ public class RulesTest {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
                 new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
 
         return request;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java

@@ -1,10 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.annotate;
 
+
 import java.util.List;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -22,19 +22,15 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.CellRectangle;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.model.SectionRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import lombok.RequiredArgsConstructor;
@@ -46,7 +42,6 @@ public class AnnotationService {
     private final RedactionStorageService redactionStorageService;
     private final DictionaryService dictionaryService;
 
-    private final RedactionLogMergeService redactionLogMergeService;
 
 
     public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
@@ -54,14 +49,8 @@ public class AnnotationService {
         var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
                 annotateRequest.getFileId(),
                 FileType.ORIGIN));
-        var mergedRedactionLog = redactionLogMergeService.provideRedactionLog(RedactionRequest.builder()
-                .fileId(annotateRequest.getFileId())
-                .manualRedactions(annotateRequest.getManualRedactions())
-                .dossierId(annotateRequest.getDossierId())
-                .dossierTemplateId(annotateRequest.getDossierTemplateId())
-                .colors(annotateRequest.getColors())
-                .types(annotateRequest.getTypes())
-                .build());
+
+        var mergedRedactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
         var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
 
         try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java

@@ -10,9 +10,9 @@ import org.springframework.core.io.ClassPathResource;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.MessageType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.utils.MetricValidationUtils;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -32,17 +32,17 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@@ -65,9 +65,6 @@ public class LiveDataIntegrationTest {
     @MockBean
     protected DictionaryClient dictionaryClient;
 
-    @MockBean
-    protected EntityRecognitionClient entityRecognitionClient;
-
     @MockBean
     private FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
 
@@ -118,6 +115,7 @@ public class LiveDataIntegrationTest {
     @BeforeEach
     public void prepareTest() {
 
+        TenantContext.setTenantId("redaction");
         when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
         when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java

@@ -7,7 +7,7 @@ import java.util.Set;
 
 import org.junit.jupiter.api.Test;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -29,12 +29,13 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
@@ -199,7 +200,7 @@ public class PdfSegmentationServiceTest {
     @SneakyThrows
     private void prepareStorage() {
 
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
                 new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
     }