Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c13d148ae7 | ||
|
|
5860daa6fb | ||
|
|
08d20e30e3 | ||
|
|
c3e0364d80 | ||
|
|
56f98549e4 | ||
|
|
5ca2e5af23 | ||
|
|
99b25a4ccb | ||
|
|
4c153fc5f8 | ||
|
|
e42ecad4e4 | ||
|
|
a73373ba42 |
@@ -12,7 +12,8 @@ mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
echo "dependency-check:aggregate"
|
||||
mvn --no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
org.owasp:dependency-check-maven:aggregate
|
||||
org.owasp:dependency-check-maven:aggregate \
|
||||
-DknownExploitedEnabled=false
|
||||
|
||||
if [[ -z "${bamboo_repository_pr_key}" ]]
|
||||
then
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
<artifactId>redaction-service</artifactId>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<version>3.0-SNAPSHOT</version>
|
||||
<version>4.0-SNAPSHOT</version>
|
||||
|
||||
|
||||
<packaging>pom</packaging>
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.20.0</version>
|
||||
<version>1.22.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
|
||||
<properties>
|
||||
<persistence-service.version>1.299.0</persistence-service.version>
|
||||
<persistence-service.version>2.1.0</persistence-service.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@@ -32,13 +32,18 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>persistence-service-api-v1</artifactId>
|
||||
<artifactId>persistence-service-internal-api-v1</artifactId>
|
||||
<version>${persistence-service.version}</version>
|
||||
<exclusions>
|
||||
|
||||
<exclusion>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>persistence-service-api-v1</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeRequest {
|
||||
|
||||
private MessageType messageType;
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
private String dossierTemplateId;
|
||||
private ManualRedactions manualRedactions;
|
||||
private OffsetDateTime lastProcessed;
|
||||
private int analysisNumber;
|
||||
|
||||
@Builder.Default
|
||||
private Set<Integer> excludedPages = new HashSet<>();
|
||||
@Builder.Default
|
||||
private Set<Integer> sectionsToReanalyse = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private List<FileAttribute> fileAttributes = new ArrayList<>();
|
||||
|
||||
}
|
||||
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeResult {
|
||||
|
||||
private MessageType messageType;
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
private long duration;
|
||||
private int numberOfPages;
|
||||
private boolean hasUpdates;
|
||||
private long dictionaryVersion;
|
||||
private long dossierDictionaryVersion;
|
||||
private long rulesVersion;
|
||||
private long legalBasisVersion;
|
||||
|
||||
private boolean wasReanalyzed;
|
||||
|
||||
private int analysisVersion;
|
||||
private int analysisNumber;
|
||||
|
||||
private ManualRedactions manualRedactions;
|
||||
|
||||
private Set<FileAttribute> addedFileAttributes;
|
||||
|
||||
}
|
||||
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class CellRectangle {
|
||||
|
||||
private Point topLeft;
|
||||
private float width;
|
||||
private float height;
|
||||
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class Change {
|
||||
|
||||
private int analysisNumber;
|
||||
private ChangeType type;
|
||||
private OffsetDateTime dateTime;
|
||||
|
||||
}
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ChangeType {
|
||||
ADDED,
|
||||
REMOVED,
|
||||
CHANGED
|
||||
}
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum Engine {
|
||||
DICTIONARY,
|
||||
NER,
|
||||
RULE
|
||||
}
|
||||
-19
@@ -1,19 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class FileAttribute {
|
||||
|
||||
private String id;
|
||||
private String label;
|
||||
private String placeholder;
|
||||
private String value;
|
||||
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ImportedRedaction {
|
||||
|
||||
private String id;
|
||||
|
||||
@Builder.Default
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ImportedRedactions {
|
||||
|
||||
@Builder.Default
|
||||
private Map<Integer, List<ImportedRedaction>> importedRedactions = new HashMap<>();
|
||||
|
||||
}
|
||||
-59
@@ -1,59 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@Builder
|
||||
public class ManualChange {
|
||||
|
||||
private AnnotationStatus annotationStatus;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
private OffsetDateTime processedDate;
|
||||
private OffsetDateTime requestedDate;
|
||||
private String userId;
|
||||
private Map<String, String> propertyChanges = new HashMap<>();
|
||||
|
||||
|
||||
public static ManualChange from(BaseAnnotation baseAnnotation) {
|
||||
|
||||
ManualChange manualChange = new ManualChange();
|
||||
manualChange.annotationStatus = baseAnnotation.getStatus();
|
||||
manualChange.processedDate = baseAnnotation.getProcessedDate();
|
||||
manualChange.requestedDate = baseAnnotation.getRequestDate();
|
||||
manualChange.userId = baseAnnotation.getUser();
|
||||
return manualChange;
|
||||
}
|
||||
|
||||
|
||||
public boolean isProcessed() {
|
||||
|
||||
return processedDate != null;
|
||||
}
|
||||
|
||||
|
||||
public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
|
||||
|
||||
this.manualRedactionType = manualRedactionType;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
public ManualChange withChange(String property, String value) {
|
||||
|
||||
this.propertyChanges.put(property, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ManualRedactionType {
|
||||
ADD_LOCALLY,
|
||||
ADD_TO_DICTIONARY,
|
||||
REMOVE_LOCALLY,
|
||||
REMOVE_FROM_DICTIONARY,
|
||||
FORCE_REDACT,
|
||||
FORCE_HINT,
|
||||
RECATEGORIZE,
|
||||
LEGAL_BASIS_CHANGE,
|
||||
RESIZE
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum MessageType {
|
||||
|
||||
ANALYSE,
|
||||
REANALYSE,
|
||||
STRUCTURE_ANALYSE,
|
||||
SURROUNDING_TEXT
|
||||
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class Point {
|
||||
|
||||
private float x;
|
||||
private float y;
|
||||
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ReanalyzeResult {
|
||||
|
||||
private RedactionLog redactionLog;
|
||||
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class Rectangle {
|
||||
|
||||
private Point topLeft;
|
||||
private float width;
|
||||
private float height;
|
||||
|
||||
private int page;
|
||||
|
||||
}
|
||||
-37
@@ -1,37 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@CompiledJson
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionLog {
|
||||
|
||||
/**
|
||||
* Version 0 Redaction Logs have manual redactions merged inside them
|
||||
* Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime.
|
||||
*/
|
||||
private long analysisVersion;
|
||||
|
||||
/**
|
||||
* Which analysis created this redactionLog.
|
||||
*/
|
||||
private int analysisNumber;
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntry = new ArrayList<>();
|
||||
private List<RedactionLogLegalBasis> legalBasis = new ArrayList<>();
|
||||
|
||||
private long dictionaryVersion = -1;
|
||||
private long dossierDictionaryVersion = -1;
|
||||
private long rulesVersion = -1;
|
||||
private long legalBasisVersion = -1;
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionLogChanges {
|
||||
|
||||
private RedactionLog redactionLog;
|
||||
private boolean hasChanges;
|
||||
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedactionLogComment {
|
||||
|
||||
private long id;
|
||||
private String user;
|
||||
private String text;
|
||||
private String annotationId;
|
||||
private String fileId;
|
||||
private OffsetDateTime date;
|
||||
private OffsetDateTime softDeletedTime;
|
||||
|
||||
}
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
|
||||
import lombok.*;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
public class RedactionLogEntry {
|
||||
|
||||
private String id;
|
||||
private String type;
|
||||
private String value;
|
||||
private String reason;
|
||||
private int matchedRule;
|
||||
private boolean rectangle;
|
||||
private String legalBasis;
|
||||
|
||||
private boolean imported;
|
||||
|
||||
private boolean redacted;
|
||||
private boolean isHint;
|
||||
private boolean isRecommendation;
|
||||
private boolean isFalsePositive;
|
||||
|
||||
private String section;
|
||||
private float[] color;
|
||||
|
||||
@Builder.Default
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
private int sectionNumber;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
@Builder.Default
|
||||
private List<RedactionLogComment> comments = new ArrayList<>();
|
||||
|
||||
private int startOffset;
|
||||
private int endOffset;
|
||||
|
||||
private boolean isImage;
|
||||
private boolean imageHasTransparency;
|
||||
|
||||
private boolean isDictionaryEntry;
|
||||
private boolean isDossierDictionaryEntry;
|
||||
|
||||
private boolean excluded;
|
||||
|
||||
private String sourceId;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
@Builder.Default
|
||||
private List<Change> changes = new ArrayList<>();
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
@Builder.Default
|
||||
private List<ManualChange> manualChanges = new ArrayList<>();
|
||||
|
||||
private Set<Engine> engines = new HashSet<>();
|
||||
|
||||
private Set<String> reference = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Set<String> importedRedactionIntersections = new HashSet<>();
|
||||
|
||||
|
||||
public boolean lastChangeIsRemoved() {
|
||||
|
||||
return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
|
||||
}
|
||||
|
||||
|
||||
public boolean isLocalManualRedaction() {
|
||||
|
||||
return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
|
||||
}
|
||||
|
||||
|
||||
public boolean isManuallyRemoved() {
|
||||
|
||||
return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
|
||||
}
|
||||
|
||||
|
||||
private <T> Optional<T> last(List<T> list) {
|
||||
|
||||
return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
|
||||
}
|
||||
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedactionLogLegalBasis {
|
||||
|
||||
private String name;
|
||||
private String description;
|
||||
private String reason;
|
||||
|
||||
}
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedactionRequest {
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
private String dossierTemplateId;
|
||||
private ManualRedactions manualRedactions;
|
||||
@Builder.Default
|
||||
private Set<Integer> excludedPages = new HashSet<>();
|
||||
|
||||
private Colors colors;
|
||||
private List<Type> types;
|
||||
|
||||
private boolean includeFalsePositives;
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedactionResult {
|
||||
|
||||
private byte[] document;
|
||||
private int numberOfPages;
|
||||
|
||||
}
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SectionArea {
|
||||
|
||||
private Point topLeft;
|
||||
private float width;
|
||||
private float height;
|
||||
private int page;
|
||||
private String header;
|
||||
|
||||
|
||||
public boolean contains(Rectangle other) {
|
||||
|
||||
return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft()
|
||||
.getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft()
|
||||
.getY() + other.getHeight();
|
||||
}
|
||||
|
||||
|
||||
// TODO we should only use one rectangle class.
|
||||
public boolean contains(com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle other) {
|
||||
|
||||
return page == other.getPage() && this.topLeft.getX() <= other.getTopLeftX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeftX() + other.getWidth() && this.getTopLeft()
|
||||
.getY() <= other.getTopLeftY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeftY() + other.getHeight();
|
||||
}
|
||||
|
||||
}
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
@Data
|
||||
@CompiledJson
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class SectionGrid {
|
||||
|
||||
private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();
|
||||
|
||||
private List<SectionGridSection> sections = new ArrayList<>();
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static class SectionGridSection {
|
||||
|
||||
private int sectionNumber;
|
||||
private String headline;
|
||||
private Set<Integer> pages;
|
||||
private List<SectionArea> sectionAreas;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class SectionRectangle {
|
||||
|
||||
private Point topLeft;
|
||||
private float width;
|
||||
private float height;
|
||||
private int part;
|
||||
private int numberOfParts;
|
||||
|
||||
private List<CellRectangle> tableCells;
|
||||
|
||||
}
|
||||
-26
@@ -1,38 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.resources;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
public interface RedactionResource {
|
||||
|
||||
@PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult classify(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
|
||||
@PostMapping(value = "/debug/sections", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult sections(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
|
||||
@PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
|
||||
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void testRules(@RequestBody String rules);
|
||||
|
||||
|
||||
@PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
|
||||
@PostMapping(value = "/manual/surrounding-text/{dossierId}/{fileId}", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions);
|
||||
|
||||
}
|
||||
|
||||
+4
-1
@@ -11,12 +11,15 @@ import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyWebConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
import io.micrometer.core.aop.TimedAspect;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
|
||||
@Import({DefaultWebMvcConfiguration.class})
|
||||
@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class})
|
||||
@EnableFeignClients(basePackageClasses = RulesClient.class)
|
||||
@EnableConfigurationProperties(RedactionServiceSettings.class)
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
+2
-2
@@ -4,8 +4,8 @@ import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.utils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
+2
-2
@@ -2,9 +2,9 @@ package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource;
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.DictionaryResource;
|
||||
|
||||
@FeignClient(name = "DictionaryResource", url = "${persistence-service.url}")
|
||||
public interface DictionaryClient extends DictionaryResource {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
|
||||
@FeignClient(name = "EntityRecognitionClient", url = "${entity-recognition-service.url}")
|
||||
public interface EntityRecognitionClient {
|
||||
|
||||
@PostMapping(value = "/find_authors", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
NerEntities findAuthors(EntityRecognitionRequest entityRecognitionRequest);
|
||||
|
||||
}
|
||||
+1
-1
@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;
|
||||
|
||||
@FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
|
||||
public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.resources.LegalBasisMappingResource;
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.LegalBasisMappingResource;
|
||||
|
||||
@FeignClient(name = "LegalBasisMappingResource", url = "${persistence-service.url}")
|
||||
public interface LegalBasisClient extends LegalBasisMappingResource {
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.resources.RulesResource;
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.RulesResource;
|
||||
|
||||
@FeignClient(name = "RulesResource", url = "${persistence-service.url}")
|
||||
public interface RulesClient extends RulesResource {
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
|
||||
|
||||
@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
|
||||
public interface TenantsClient extends TenantsResource {
|
||||
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class EntityRecognitionRequest {
|
||||
|
||||
private List<EntityRecognitionSection> data;
|
||||
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class EntityRecognitionResult {
|
||||
|
||||
@Builder.Default
|
||||
private Map<Integer, List<EntityRecogintionEntity>> entities = new HashMap<>();
|
||||
|
||||
}
|
||||
-139
@@ -1,31 +1,11 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -35,100 +15,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionController implements RedactionResource {
|
||||
|
||||
private final PdfVisualisationService pdfVisualisationService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final PdfSegmentationService pdfSegmentationService;
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
private final RedactionLogMergeService redactionLogMergeService;
|
||||
private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
|
||||
redactionRequest.getFileId(),
|
||||
FileType.ORIGIN));
|
||||
try {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
|
||||
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
|
||||
redactionRequest.getFileId(),
|
||||
FileType.ORIGIN));
|
||||
try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
|
||||
redactionRequest.getFileId(),
|
||||
FileType.ORIGIN));
|
||||
try {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
|
||||
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
|
||||
redactionRequest.getFileId(),
|
||||
FileType.ORIGIN));
|
||||
try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
Document classifiedDoc;
|
||||
|
||||
try {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
|
||||
redactionRequest.getFileId(),
|
||||
FileType.ORIGIN));
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Page page : classifiedDoc.getPages()) {
|
||||
for (AbstractTextContainer textContainer : page.getTextBlocks()) {
|
||||
if (textContainer instanceof Table) {
|
||||
Table table = (Table) textContainer;
|
||||
sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return RedactionResult.builder().document(sb.toString().getBytes()).build();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@@ -141,30 +28,4 @@ public class RedactionController implements RedactionResource {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
|
||||
|
||||
return redactionLogMergeService.provideRedactionLog(redactionRequest);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
|
||||
|
||||
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
document.save(byteArrayOutputStream);
|
||||
return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions) {
|
||||
|
||||
var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
|
||||
log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());
|
||||
return result.getManualRedactions();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import java.util.concurrent.Executor;
|
||||
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
|
||||
@Configuration
|
||||
public class AsyncConfig extends AsyncConfigurerSupport {
|
||||
|
||||
@Override
|
||||
public Executor getAsyncExecutor() {
|
||||
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
|
||||
executor.setCorePoolSize(7);
|
||||
executor.setMaxPoolSize(42);
|
||||
executor.setQueueCapacity(11);
|
||||
executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
|
||||
executor.setTaskDecorator(new TenantAwareTaskDecorator());
|
||||
executor.initialize();
|
||||
|
||||
return executor;
|
||||
}
|
||||
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.spec.KeySpec;
|
||||
import java.util.Base64;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.SecretKey;
|
||||
import javax.crypto.SecretKeyFactory;
|
||||
import javax.crypto.spec.GCMParameterSpec;
|
||||
import javax.crypto.spec.PBEKeySpec;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@Service
|
||||
public class EncryptionDecryptionService {
|
||||
|
||||
@Value("${redaction-service.crypto.key:redaction}")
|
||||
private String key;
|
||||
|
||||
private SecretKey secretKey;
|
||||
private byte[] iv;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@PostConstruct
|
||||
protected void postConstruct() {
|
||||
|
||||
SecureRandom secureRandom = new SecureRandom();
|
||||
iv = new byte[12];
|
||||
secureRandom.nextBytes(iv);
|
||||
secretKey = generateSecretKey(key, iv);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public String encrypt(String strToEncrypt) {
|
||||
|
||||
return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public String decrypt(String strToDecrypt) {
|
||||
|
||||
byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
|
||||
return new String(decrypt(bytes), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public byte[] encrypt(byte[] data) {
|
||||
|
||||
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
|
||||
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
|
||||
byte[] encryptedData = cipher.doFinal(data);
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
|
||||
byteBuffer.putInt(iv.length);
|
||||
byteBuffer.put(iv);
|
||||
byteBuffer.put(encryptedData);
|
||||
return byteBuffer.array();
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public byte[] decrypt(byte[] encryptedData) {
|
||||
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
|
||||
int noonceSize = byteBuffer.getInt();
|
||||
if (noonceSize < 12 || noonceSize >= 16) {
|
||||
throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
|
||||
}
|
||||
byte[] iv = new byte[noonceSize];
|
||||
byteBuffer.get(iv);
|
||||
|
||||
SecretKey secretKey = generateSecretKey(key, iv);
|
||||
|
||||
byte[] cipherBytes = new byte[byteBuffer.remaining()];
|
||||
byteBuffer.get(cipherBytes);
|
||||
|
||||
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
|
||||
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
|
||||
return cipher.doFinal(cipherBytes);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public SecretKey generateSecretKey(String password, byte[] iv) {
|
||||
|
||||
KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
|
||||
SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
|
||||
byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
|
||||
return new SecretKeySpec(key, "AES");
|
||||
}
|
||||
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import feign.RequestInterceptor;
|
||||
import feign.RequestTemplate;
|
||||
|
||||
@Component
|
||||
public class ForwardTenantInterceptor implements RequestInterceptor {
|
||||
|
||||
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
|
||||
|
||||
@Override
|
||||
public void apply(RequestTemplate template) {
|
||||
template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME;
|
||||
|
||||
import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.config.BeanPostProcessor;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class MultiTenancyMessagingConfiguration {
|
||||
|
||||
@Bean
|
||||
public static BeanPostProcessor multitenancyBeanPostProcessor() {
|
||||
|
||||
return new BeanPostProcessor() {
|
||||
|
||||
@Override
|
||||
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
|
||||
|
||||
if (bean instanceof RabbitTemplate) {
|
||||
|
||||
((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
|
||||
m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId());
|
||||
return m;
|
||||
});
|
||||
|
||||
} else if (bean instanceof AbstractRabbitListenerContainerFactory) {
|
||||
|
||||
((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
|
||||
String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME);
|
||||
|
||||
if (tenant != null) {
|
||||
TenantContext.setTenantId(tenant);
|
||||
} else {
|
||||
throw new RuntimeException("No Tenant is set queue message");
|
||||
}
|
||||
return m;
|
||||
});
|
||||
}
|
||||
return bean;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
|
||||
@Configuration
|
||||
public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
|
||||
|
||||
private final TenantInterceptor tenantInterceptor;
|
||||
|
||||
|
||||
@Autowired
|
||||
public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
|
||||
|
||||
this.tenantInterceptor = tenantInterceptor;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void addInterceptors(InterceptorRegistry registry) {
|
||||
|
||||
registry.addWebRequestInterceptor(tenantInterceptor);
|
||||
}
|
||||
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.client.TenantsClient;
|
||||
import com.iqser.red.storage.commons.model.AzureStorageConnection;
|
||||
import com.iqser.red.storage.commons.model.S3StorageConnection;
|
||||
import com.iqser.red.storage.commons.service.StorageConnectionProvider;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class StorageConnectionProviderImpl implements StorageConnectionProvider {
|
||||
|
||||
private final TenantsClient tenantsClient;
|
||||
private final EncryptionDecryptionService encryptionDecryptionService;
|
||||
|
||||
|
||||
@Override
|
||||
public AzureStorageConnection getAzureStorageConnection(String tenantId) {
|
||||
|
||||
var tenant = tenantsClient.getTenant(tenantId);
|
||||
return AzureStorageConnection.builder()
|
||||
.connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
|
||||
.containerName(tenant.getAzureStorageConnection().getContainerName())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public S3StorageConnection getS3StorageConnection(String tenantId) {
|
||||
|
||||
var tenant = tenantsClient.getTenant(tenantId);
|
||||
return S3StorageConnection.builder()
|
||||
.key(tenant.getS3StorageConnection().getKey())
|
||||
.secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
|
||||
.signerType(tenant.getS3StorageConnection().getSignerType())
|
||||
.bucketName(tenant.getS3StorageConnection().getBucketName())
|
||||
.region(tenant.getS3StorageConnection().getRegion())
|
||||
.endpoint(tenant.getS3StorageConnection().getEndpoint())
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.core.task.TaskDecorator;
|
||||
import org.springframework.lang.NonNull;
|
||||
|
||||
public class TenantAwareTaskDecorator implements TaskDecorator {
|
||||
|
||||
@Override
|
||||
@NonNull
|
||||
public Runnable decorate(@NonNull Runnable runnable) {
|
||||
|
||||
String tenantId = TenantContext.getTenantId();
|
||||
return () -> {
|
||||
try {
|
||||
TenantContext.setTenantId(tenantId);
|
||||
runnable.run();
|
||||
} finally {
|
||||
TenantContext.setTenantId(null);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public final class TenantContext {
|
||||
|
||||
private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
|
||||
|
||||
|
||||
public static void setTenantId(String tenantId) {
|
||||
|
||||
log.debug("Setting tenantId to " + tenantId);
|
||||
currentTenant.set(tenantId);
|
||||
}
|
||||
|
||||
|
||||
public static String getTenantId() {
|
||||
|
||||
return currentTenant.get();
|
||||
}
|
||||
|
||||
|
||||
public static void clear() {
|
||||
|
||||
currentTenant.remove();
|
||||
}
|
||||
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.ui.ModelMap;
|
||||
import org.springframework.web.context.request.WebRequest;
|
||||
import org.springframework.web.context.request.WebRequestInterceptor;
|
||||
|
||||
@Component
|
||||
public class TenantInterceptor implements WebRequestInterceptor {
|
||||
|
||||
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
|
||||
|
||||
|
||||
@Override
|
||||
public void preHandle(WebRequest request) {
|
||||
|
||||
if (request.getHeader(TENANT_HEADER_NAME) != null) {
|
||||
TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void postHandle(WebRequest request, ModelMap model) {
|
||||
|
||||
TenantContext.clear();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void afterCompletion(WebRequest request, Exception ex) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -12,8 +12,8 @@ import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
|
||||
+2
-2
@@ -10,8 +10,8 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.*;
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
|
||||
+4
-4
@@ -18,11 +18,11 @@ import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.model.ArgumentType;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
|
||||
+7
-7
@@ -5,11 +5,11 @@ import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.ImportedRedaction;
|
||||
import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
@@ -51,8 +51,8 @@ public class ImportedRedactionService {
|
||||
List<RedactionLogEntry> redactionLogEntries,
|
||||
ImportedRedactions importedRedactions) {
|
||||
|
||||
for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
|
||||
for (ImportedRedaction importedRedaction : importedRedactionsValues) {
|
||||
for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
|
||||
for (var importedRedaction : importedRedactionsValues) {
|
||||
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
|
||||
.id(importedRedaction.getId())
|
||||
.type(IMPORTED_REDACTION_TYPE)
|
||||
|
||||
+7
-7
@@ -7,12 +7,11 @@ import java.util.Set;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
@@ -23,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -125,7 +125,7 @@ public class ManualRedactionSurroundingTextService {
|
||||
}
|
||||
|
||||
|
||||
public boolean intersects(Rectangle manualPosition, com.iqser.red.service.redaction.v1.model.Rectangle textPositionRectangle) {
|
||||
public boolean intersects(Rectangle manualPosition, com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle textPositionRectangle) {
|
||||
|
||||
return textPositionRectangle.getTopLeft().getX() + textPositionRectangle.getWidth() > manualPosition.getTopLeftX() && textPositionRectangle.getTopLeft()
|
||||
.getY() + textPositionRectangle.getHeight() > manualPosition.getTopLeftY() && textPositionRectangle.getTopLeft()
|
||||
|
||||
+5
-1
@@ -1,6 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
|
||||
+3
-3
@@ -9,9 +9,9 @@ import java.util.Set;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
|
||||
-545
@@ -1,545 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.Change;
|
||||
import com.iqser.red.service.redaction.v1.model.ChangeType;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualChange;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionLogMergeService {
|
||||
|
||||
private static final String DELETED_TYPE_COLOR = "#9398a0";
|
||||
private final SectionTextService sectionTextService;
|
||||
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
|
||||
|
||||
@Timed("redactmanager_getMergedRedactionLog")
|
||||
public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
|
||||
|
||||
log.debug("Requested preview for: {}", redactionRequest);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
|
||||
|
||||
if (redactionLog == null) {
|
||||
throw new NotFoundException("RedactionLog not present");
|
||||
}
|
||||
|
||||
SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
|
||||
if (sectionGrid.getSections().isEmpty()) {
|
||||
|
||||
log.info("SectionGrid does not have headlines set. Computing headlines now!");
|
||||
var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
|
||||
|
||||
// enhance section grid with headline data
|
||||
for (var sectionText : text.getSectionTexts()) {
|
||||
sectionGrid.getSections()
|
||||
.add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
|
||||
sectionText.getHeadline(),
|
||||
sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
|
||||
sectionText.getSectionAreas()));
|
||||
}
|
||||
redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
|
||||
}
|
||||
|
||||
log.debug("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
|
||||
var merged = mergeRedactionLogData(redactionLog,
|
||||
sectionGrid,
|
||||
redactionRequest.getManualRedactions(),
|
||||
redactionRequest.getExcludedPages(),
|
||||
redactionRequest.getTypes(),
|
||||
redactionRequest.getColors());
|
||||
|
||||
merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
|
||||
|
||||
return merged;
|
||||
}
|
||||
|
||||
|
||||
private RedactionLog mergeRedactionLogData(RedactionLog redactionLog,
|
||||
SectionGrid sectionGrid,
|
||||
ManualRedactions manualRedactions,
|
||||
Set<Integer> excludedPages,
|
||||
List<Type> types,
|
||||
Colors colors) {
|
||||
|
||||
var skippedImportedRedactions = new HashSet<>();
|
||||
log.info("Merging Redaction log with manual redactions");
|
||||
if (manualRedactions != null) {
|
||||
|
||||
var manualRedactionLogEntries = addManualAddEntries(sectionGrid,
|
||||
manualRedactions.getEntriesToAdd(),
|
||||
manualRedactions.getComments(),
|
||||
colors,
|
||||
types,
|
||||
redactionLog.getAnalysisNumber());
|
||||
|
||||
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
|
||||
|
||||
var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
|
||||
|
||||
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
|
||||
|
||||
processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())).collect(Collectors.toList()), entry, types, colors);
|
||||
|
||||
if (entry.isImported() && !entry.isRedacted()) {
|
||||
skippedImportedRedactions.add(entry.getId());
|
||||
}
|
||||
|
||||
entry.setComments(convert(manualRedactions.getComments().get(entry.getId())));
|
||||
|
||||
if (excludedPages != null && !excludedPages.isEmpty()) {
|
||||
entry.getPositions().forEach(pos -> {
|
||||
if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
|
||||
entry.setExcluded(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
redactionLog.getRedactionLogEntry().removeIf(entry -> {
|
||||
|
||||
if (entry.getImportedRedactionIntersections() != null) {
|
||||
entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
|
||||
if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
|
||||
.equals("ocr")))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (processedIds.contains(entry.getId())) {
|
||||
log.info("Duplicate annotation found with id {}", entry.getId());
|
||||
return true;
|
||||
}
|
||||
processedIds.add(entry.getId());
|
||||
return false;
|
||||
});
|
||||
return redactionLog;
|
||||
}
|
||||
|
||||
|
||||
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
|
||||
|
||||
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
|
||||
|
||||
manualRedactions.getImageRecategorization().forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
|
||||
}
|
||||
});
|
||||
|
||||
manualRedactions.getIdsToRemove().forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
|
||||
}
|
||||
});
|
||||
|
||||
manualRedactions.getForceRedactions().forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
|
||||
}
|
||||
});
|
||||
|
||||
manualRedactions.getLegalBasisChanges().forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
|
||||
}
|
||||
});
|
||||
|
||||
manualRedactions.getResizeRedactions().forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
|
||||
}
|
||||
});
|
||||
|
||||
Collections.sort(manualRedactionWrappers);
|
||||
|
||||
return manualRedactionWrappers;
|
||||
}
|
||||
|
||||
|
||||
private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors) {
|
||||
|
||||
manualRedactionWrappers.forEach(mrw -> {
|
||||
|
||||
if (mrw.getItem() instanceof ManualImageRecategorization) {
|
||||
var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
|
||||
String manualOverrideReason = null;
|
||||
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
|
||||
redactionLogEntry.setType(imageRecategorization.getType());
|
||||
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
|
||||
|
||||
if (isHint(types, imageRecategorization.getType())) {
|
||||
redactionLogEntry.setRedacted(false);
|
||||
redactionLogEntry.setHint(true);
|
||||
} else {
|
||||
redactionLogEntry.setHint(false);
|
||||
}
|
||||
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
|
||||
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
|
||||
}
|
||||
|
||||
if (manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(imageRecategorization)
|
||||
.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
|
||||
.withChange("type", imageRecategorization.getType()));
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof IdRemoval) {
|
||||
var manualRemoval = (IdRemoval) mrw.getItem();
|
||||
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
|
||||
log.debug("Skipping merge for dictionary-modifying entry");
|
||||
} else {
|
||||
String manualOverrideReason = null;
|
||||
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
redactionLogEntry.setRedacted(false);
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), true, types));
|
||||
redactionLogEntry.setHint(false);
|
||||
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
|
||||
}
|
||||
|
||||
if (manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(manualRemoval)
|
||||
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof ManualForceRedaction) {
|
||||
var manualForceRedact = (ManualForceRedaction) mrw.getItem();
|
||||
String manualOverrideReason = null;
|
||||
var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
|
||||
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
// Forcing a skipped hint should result in a hint
|
||||
if (dictionaryIsHint) {
|
||||
redactionLogEntry.setHint(true);
|
||||
} else {
|
||||
redactionLogEntry.setRedacted(true);
|
||||
}
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
}
|
||||
|
||||
if (manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
|
||||
var manualChange = ManualChange.from(manualForceRedact)
|
||||
.withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
|
||||
|
||||
redactionLogEntry.getManualChanges().add(manualChange);
|
||||
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof ManualLegalBasisChange) {
|
||||
var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
|
||||
String manualOverrideReason = null;
|
||||
if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
|
||||
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
|
||||
redactionLogEntry.setRedacted(true);
|
||||
if (manualLegalBasisChange.getSection() != null) {
|
||||
redactionLogEntry.setSection(manualLegalBasisChange.getSection());
|
||||
}
|
||||
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
|
||||
redactionLogEntry.setValue(manualLegalBasisChange.getValue());
|
||||
}
|
||||
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
|
||||
}
|
||||
|
||||
if (manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
|
||||
var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
|
||||
manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
|
||||
if (manualLegalBasisChange.getSection() != null) {
|
||||
manualChange.withChange("section", manualLegalBasisChange.getSection());
|
||||
}
|
||||
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
|
||||
manualChange.withChange("value", manualLegalBasisChange.getValue());
|
||||
}
|
||||
redactionLogEntry.getManualChanges().add(manualChange);
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof ManualResizeRedaction) {
|
||||
var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
|
||||
String manualOverrideReason = null;
|
||||
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
|
||||
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
|
||||
if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
|
||||
redactionLogEntry.setValue(manualResizeRedact.getValue());
|
||||
}
|
||||
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
|
||||
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
}
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
|
||||
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
|
||||
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
|
||||
|
||||
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
|
||||
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
private String mergeReasonIfNecessary(String currentReason, String addition) {
|
||||
|
||||
if (currentReason != null) {
|
||||
if (!currentReason.contains(addition)) {
|
||||
return currentReason + addition;
|
||||
}
|
||||
return currentReason;
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
|
||||
Set<ManualRedactionEntry> manualAdds,
|
||||
Map<String, List<Comment>> comments,
|
||||
Colors colors,
|
||||
List<Type> types,
|
||||
int analysisNumber) {
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
|
||||
|
||||
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
|
||||
|
||||
if (shouldCreateManualEntry(manualRedactionEntry)) {
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types, analysisNumber);
|
||||
redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
|
||||
redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId())));
|
||||
redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
|
||||
|
||||
sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
|
||||
|
||||
redactionLogEntries.add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
|
||||
return redactionLogEntries;
|
||||
}
|
||||
|
||||
|
||||
private List<RedactionLogComment> convert(List<Comment> comments) {
|
||||
|
||||
return comments == null ? null : comments.stream()
|
||||
.map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions) {
|
||||
|
||||
return positions.stream()
|
||||
.map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("PMD.UselessParentheses")
|
||||
private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, Colors colors, List<Type> types, int analysisNumber) {
|
||||
|
||||
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
|
||||
|
||||
var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
|
||||
List<ManualChange> changeList = new ArrayList<>();
|
||||
changeList.add(change);
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.color(getColorForManualAdd(manualRedactionEntry.getType(), colors, manualRedactionEntry.getStatus(), types))
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
|
||||
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.sourceId(manualRedactionEntry.getSourceId())
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.redacted(true)
|
||||
.isHint(false)
|
||||
.sectionNumber(-1)
|
||||
.rectangle(manualRedactionEntry.isRectangle())
|
||||
.manualChanges(changeList)
|
||||
.changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped, List<Type> types) {
|
||||
|
||||
if (requested) {
|
||||
return convertColor(colors.getRequestRemoveColor());
|
||||
}
|
||||
if (skipped || !isRedaction && !isHint(types, type)) {
|
||||
return convertColor(colors.getSkippedColor());
|
||||
}
|
||||
return getColor(types, type);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
|
||||
|
||||
if (status.equals(AnnotationStatus.REQUESTED)) {
|
||||
return convertColor(colors.getRequestAddColor());
|
||||
} else if (status.equals(AnnotationStatus.DECLINED)) {
|
||||
return convertColor(colors.getSkippedColor());
|
||||
}
|
||||
return getColor(types, type);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(List<Type> types, String type) {
|
||||
|
||||
var matchingTypes = getMatchingTypes(types, type);
|
||||
Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
|
||||
if (foundAndNotDeletedType.isPresent()) {
|
||||
return convertColor(foundAndNotDeletedType.get().getHexColor());
|
||||
}
|
||||
Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
|
||||
return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor(DELETED_TYPE_COLOR));
|
||||
}
|
||||
|
||||
|
||||
boolean isHint(List<Type> types, String type) {
|
||||
|
||||
var matchingTypes = getMatchingTypes(types, type);
|
||||
Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
|
||||
if (foundAndNotDeletedType.isPresent()) {
|
||||
return foundAndNotDeletedType.get().isHint();
|
||||
}
|
||||
Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
|
||||
return firstDeletedType.map(Type::isHint).orElse(false);
|
||||
}
|
||||
|
||||
|
||||
private float[] convertColor(String hex) {
|
||||
|
||||
Color color = Color.decode(hex);
|
||||
return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
|
||||
}
|
||||
|
||||
|
||||
private List<Type> getMatchingTypes(List<Type> types, String type) {
|
||||
|
||||
return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
private boolean isDeletedType(Type type) {
|
||||
|
||||
return type.getSoftDeletedTime() != null;
|
||||
}
|
||||
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
|
||||
|
||||
private String id;
|
||||
private OffsetDateTime date;
|
||||
private Object item;
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(ManualRedactionWrapper o) {
|
||||
|
||||
return this.date.compareTo(o.date);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-3
@@ -1,8 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.CellRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import java.util.stream.Collectors;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
|
||||
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class SectionTextService {
|
||||
|
||||
public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
if (redactionLogEntry.getSection() != null) {
|
||||
// set by UI
|
||||
return;
|
||||
}
|
||||
|
||||
if (sectionGrid != null) {
|
||||
var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null;
|
||||
|
||||
if (firstPosition != null) {
|
||||
|
||||
for (var section : sectionGrid.getSections()) {
|
||||
if (section.getPages().contains(firstPosition.getPage())) {
|
||||
for (var sectionArea : section.getSectionAreas()) {
|
||||
if (sectionArea.contains(firstPosition)) {
|
||||
redactionLogEntry.setSection(section.getHeadline());
|
||||
redactionLogEntry.setSectionNumber(section.getSectionNumber());
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+10
-10
@@ -15,16 +15,16 @@ import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
|
||||
+10
-10
@@ -9,16 +9,16 @@ import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
|
||||
|
||||
+2
-2
@@ -8,8 +8,8 @@ import java.util.stream.Collectors;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
|
||||
+5
-5
@@ -12,11 +12,11 @@ import java.util.stream.Stream;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import java.util.Map;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@ import java.util.Map;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableCells;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableServiceResponse;
|
||||
@@ -60,4 +60,4 @@ public class TableService {
|
||||
return pdfTableCells;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+13
-12
@@ -4,13 +4,14 @@ import java.io.InputStream;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
@@ -31,14 +32,14 @@ public class RedactionStorageService {
|
||||
@SneakyThrows
|
||||
public InputStream getStoredObject(String storageId) {
|
||||
|
||||
return storageService.getObject(storageId).getInputStream();
|
||||
return storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream();
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) {
|
||||
|
||||
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
|
||||
storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
|
||||
}
|
||||
|
||||
|
||||
@@ -46,7 +47,7 @@ public class RedactionStorageService {
|
||||
@Timed("redactmanager_storeObject")
|
||||
public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
|
||||
|
||||
storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
|
||||
storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
|
||||
}
|
||||
|
||||
|
||||
@@ -54,7 +55,7 @@ public class RedactionStorageService {
|
||||
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
|
||||
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Imported redactions not available.");
|
||||
return null;
|
||||
@@ -66,7 +67,7 @@ public class RedactionStorageService {
|
||||
public RedactionLog getRedactionLog(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
|
||||
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Text not available.");
|
||||
return null;
|
||||
@@ -79,7 +80,7 @@ public class RedactionStorageService {
|
||||
public Text getText(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
|
||||
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Text not available.");
|
||||
return null;
|
||||
@@ -91,7 +92,7 @@ public class RedactionStorageService {
|
||||
public NerEntities getNerEntities(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
|
||||
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
throw new NotFoundException("NER Entities are not available.");
|
||||
}
|
||||
@@ -102,7 +103,7 @@ public class RedactionStorageService {
|
||||
public SectionGrid getSectionGrid(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
|
||||
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
throw new NotFoundException("Section Grid is not available.");
|
||||
}
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
|
||||
|
||||
+8
-2
@@ -1,11 +1,17 @@
|
||||
server:
|
||||
port: 8083
|
||||
|
||||
configuration-service.url: "http://localhost:8081"
|
||||
file-management-service.url: "http://localhost:8085"
|
||||
persistence-service.url: "http://localhost:8085"
|
||||
|
||||
storage:
|
||||
bucket-name: 'redaction'
|
||||
endpoint: 'http://localhost:9000'
|
||||
key: minioadmin
|
||||
secret: minioadmin
|
||||
|
||||
|
||||
redaction-service:
|
||||
enableImageClassification: false
|
||||
cvTableParsingEnabled: false
|
||||
nerServiceEnabled: false
|
||||
priorityMode: false
|
||||
|
||||
@@ -38,8 +38,4 @@ management:
|
||||
|
||||
|
||||
storage:
|
||||
signer-type: 'AWSS3V4SignerType'
|
||||
bucket-name: 'redaction'
|
||||
region: 'us-east-1'
|
||||
endpoint: 'https://s3.amazonaws.com'
|
||||
backend: 's3'
|
||||
|
||||
+4
-3
@@ -23,9 +23,9 @@ import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
@@ -75,6 +75,7 @@ public class DictionaryServiceTest {
|
||||
"#cccccc",
|
||||
"#cccccc",
|
||||
"#cccccc",
|
||||
"#cccccc",
|
||||
"#cccccc"));
|
||||
|
||||
var type1 = new Type();
|
||||
|
||||
+6
-12
@@ -33,7 +33,7 @@ public class FileSystemBackedStorageService implements StorageService {
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public InputStreamResource getObject(String objectId) {
|
||||
public InputStreamResource getObject(String tenantId, String objectId) {
|
||||
|
||||
var res = dataMap.get(objectId);
|
||||
if (res == null) {
|
||||
@@ -45,28 +45,22 @@ public class FileSystemBackedStorageService implements StorageService {
|
||||
|
||||
|
||||
@Override
|
||||
public void deleteObject(String objectId) {
|
||||
public void deleteObject(String tenantId, String objectId) {
|
||||
|
||||
dataMap.remove(objectId);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean objectExists(String objectId) {
|
||||
public boolean objectExists(String tenantId, String objectId) {
|
||||
|
||||
return dataMap.containsKey(objectId);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public <T> void storeJSONObject(String objectId, T any) {
|
||||
public <T> void storeJSONObject(String tenantId, String objectId, T any) {
|
||||
|
||||
File tempFile = FileUtils.createTempFile("test", ".tmp");
|
||||
getMapper().writeValue(new FileOutputStream(tempFile), any);
|
||||
@@ -82,7 +76,7 @@ public class FileSystemBackedStorageService implements StorageService {
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public <T> T readJSONObject(String objectId, Class<T> clazz) {
|
||||
public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
|
||||
|
||||
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
|
||||
throw new StorageObjectDoesNotExist("Stored object not found");
|
||||
@@ -105,7 +99,7 @@ public class FileSystemBackedStorageService implements StorageService {
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void storeObject(String objectId, InputStream stream) {
|
||||
public void storeObject(String tenantId, String objectId, InputStream stream) {
|
||||
|
||||
File tempFile = FileUtils.createTempFile("test", ".tmp");
|
||||
|
||||
|
||||
+11
-12
@@ -42,23 +42,22 @@ import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.ChangeType;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
@@ -362,8 +361,8 @@ public class HeadlinesGoldStandardIntegrationTest {
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
|
||||
|
||||
return request;
|
||||
}
|
||||
@@ -392,7 +391,7 @@ public class HeadlinesGoldStandardIntegrationTest {
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
|
||||
+26
-133
@@ -51,27 +51,25 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
|
||||
@@ -81,6 +79,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||
@@ -424,12 +423,7 @@ public class RedactionIntegrationTest {
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
var mergedRedactionLog = redactionController.getRedactionLog(RedactionRequest.builder()
|
||||
.manualRedactions(manualRedactions)
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
var mergedRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
|
||||
assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
|
||||
@@ -648,7 +642,7 @@ public class RedactionIntegrationTest {
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
@@ -1106,109 +1100,8 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void classificationTest() throws IOException {
|
||||
|
||||
System.out.println("classificationTest");
|
||||
|
||||
AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
|
||||
|
||||
RedactionRequest redactionRequest = RedactionRequest.builder()
|
||||
.dossierId(request.getDossierId())
|
||||
.fileId(request.getFileId())
|
||||
.dossierTemplateId(request.getDossierTemplateId())
|
||||
.build();
|
||||
|
||||
RedactionResult result = redactionController.classify(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void classificationTestWithCvTableService() throws IOException {
|
||||
|
||||
System.out.println("classificationTest");
|
||||
|
||||
String tableServiceResponseFile = "files/cv_table_response_VV-511309.json";
|
||||
AnalyzeRequest request = prepareStorage("files/new/VV-511309_OCR.pdf", tableServiceResponseFile);
|
||||
|
||||
RedactionRequest redactionRequest = RedactionRequest.builder()
|
||||
.dossierId(request.getDossierId())
|
||||
.fileId(request.getFileId())
|
||||
.dossierTemplateId(request.getDossierTemplateId())
|
||||
.build();
|
||||
|
||||
RedactionResult result = redactionController.classify(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void sectionsTest() throws IOException {
|
||||
|
||||
System.out.println("sectionsTest");
|
||||
|
||||
AnalyzeRequest request = prepareStorage("files/Minimal Examples/Single Table.pdf");
|
||||
|
||||
RedactionRequest redactionRequest = RedactionRequest.builder()
|
||||
.dossierId(request.getDossierId())
|
||||
.fileId(request.getFileId())
|
||||
.dossierTemplateId(request.getDossierTemplateId())
|
||||
.build();
|
||||
|
||||
RedactionResult result = redactionController.sections(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Sections.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void htmlTablesTest() throws IOException {
|
||||
|
||||
System.out.println("htmlTablesTest");
|
||||
AnalyzeRequest request = prepareStorage("files/new/table-with-merged-cells.pdf");
|
||||
|
||||
RedactionRequest redactionRequest = RedactionRequest.builder()
|
||||
.dossierId(request.getDossierId())
|
||||
.fileId(request.getFileId())
|
||||
.dossierTemplateId(request.getDossierTemplateId())
|
||||
.build();
|
||||
|
||||
RedactionResult result = redactionController.htmlTables(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void htmlTableRotationTest() throws IOException {
|
||||
|
||||
System.out.println("htmlTableRotationTest");
|
||||
|
||||
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||
|
||||
RedactionRequest redactionRequest = RedactionRequest.builder()
|
||||
.dossierId(request.getDossierId())
|
||||
.fileId(request.getFileId())
|
||||
.dossierTemplateId(request.getDossierTemplateId())
|
||||
.build();
|
||||
|
||||
RedactionResult result = redactionController.htmlTables(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@@ -1638,7 +1531,7 @@ public class RedactionIntegrationTest {
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@@ -1704,7 +1597,7 @@ public class RedactionIntegrationTest {
|
||||
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
|
||||
|
||||
AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
|
||||
importedRedactions.getInputStream());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
@@ -1775,8 +1668,8 @@ public class RedactionIntegrationTest {
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
|
||||
|
||||
return request;
|
||||
|
||||
|
||||
+15
-20
@@ -52,30 +52,27 @@ import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.Change;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualChange;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
@@ -224,8 +221,6 @@ public class RulesTest {
|
||||
@Autowired
|
||||
private RedactionController redactionController;
|
||||
@Autowired
|
||||
private AnnotationService annotationService;
|
||||
@Autowired
|
||||
private AnalyzeService analyzeService;
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
@@ -624,7 +619,7 @@ public class RulesTest {
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@@ -647,9 +642,9 @@ public class RulesTest {
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
|
||||
new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
|
||||
|
||||
return request;
|
||||
|
||||
|
||||
+4
-3
@@ -1,10 +1,11 @@
|
||||
package com.iqser.red.service.redaction.v1.server.annotate;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
|
||||
+9
-20
@@ -22,19 +22,15 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.CellRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -46,7 +42,6 @@ public class AnnotationService {
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
private final RedactionLogMergeService redactionLogMergeService;
|
||||
|
||||
|
||||
public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
|
||||
@@ -54,14 +49,8 @@ public class AnnotationService {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
|
||||
annotateRequest.getFileId(),
|
||||
FileType.ORIGIN));
|
||||
var mergedRedactionLog = redactionLogMergeService.provideRedactionLog(RedactionRequest.builder()
|
||||
.fileId(annotateRequest.getFileId())
|
||||
.manualRedactions(annotateRequest.getManualRedactions())
|
||||
.dossierId(annotateRequest.getDossierId())
|
||||
.dossierTemplateId(annotateRequest.getDossierTemplateId())
|
||||
.colors(annotateRequest.getColors())
|
||||
.types(annotateRequest.getTypes())
|
||||
.build());
|
||||
|
||||
var mergedRedactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
|
||||
var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
|
||||
|
||||
+3
-3
@@ -10,9 +10,9 @@ import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.MessageType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.MetricValidationUtils;
|
||||
|
||||
|
||||
+4
-8
@@ -32,14 +32,13 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
@@ -65,9 +64,6 @@ public class LiveDataIntegrationTest {
|
||||
@MockBean
|
||||
protected DictionaryClient dictionaryClient;
|
||||
|
||||
@MockBean
|
||||
protected EntityRecognitionClient entityRecognitionClient;
|
||||
|
||||
@MockBean
|
||||
private FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
|
||||
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
|
||||
|
||||
+3
-2
@@ -29,12 +29,13 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
@@ -199,7 +200,7 @@ public class PdfSegmentationServiceTest {
|
||||
@SneakyThrows
|
||||
private void prepareStorage() {
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
|
||||
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
|
||||
new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
|
||||
}
|
||||
|
||||
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user