diff --git a/redaction-service-image-v1/src/main/docker/Dockerfile b/redaction-service-image-v1/src/main/docker/Dockerfile index 3ab50727..dec2d219 100644 --- a/redaction-service-image-v1/src/main/docker/Dockerfile +++ b/redaction-service-image-v1/src/main/docker/Dockerfile @@ -13,6 +13,6 @@ RUN apt-get update \ wget cabextract xfonts-utils fonts-liberation \ && rm -rf /var/lib/apt/lists/* -RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.7_all.deb -o /tmp/ttf-mscorefonts-installer_3.7_all.deb \ - && dpkg -i /tmp/ttf-mscorefonts-installer_3.7_all.deb \ - && rm /tmp/ttf-mscorefonts-installer_3.7_all.deb \ +RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb -o /tmp/ttf-mscorefonts-installer_3.8_all.deb \ + && dpkg -i /tmp/ttf-mscorefonts-installer_3.8_all.deb \ + && rm /tmp/ttf-mscorefonts-installer_3.8_all.deb \ diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 7a891277..8f287ddd 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -7,9 +7,8 @@ import lombok.NoArgsConstructor; import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; @Data @@ -24,7 +23,11 @@ public class AnalyzeRequest { private boolean reanalyseOnlyIfPossible; private ManualRedactions manualRedactions; private OffsetDateTime lastProcessed; - private Set excludedPages; + + @Builder.Default + private Set excludedPages = new HashSet<>(); + @Builder.Default + private Set sectionsToReanalyse = new HashSet<>(); @Builder.Default private List fileAttributes = new ArrayList<>(); diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java index 5eadb70f..4fc3fada 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java @@ -15,10 +15,6 @@ public class AnalyzeResult { private String fileId; private long duration; private int numberOfPages; - private boolean hasHints; - private boolean hasRequests; - private boolean hasRedactions; - private boolean hasImages; private boolean hasUpdates; private long dictionaryVersion; private long dossierDictionaryVersion; @@ -27,6 +23,8 @@ public class AnalyzeResult { private boolean wasReanalyzed; + private int analysisVersion; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java new file mode 100644 index 00000000..ebc59a84 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.model; + +import java.time.OffsetDateTime; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class Change { + + private ChangeType type; + private OffsetDateTime dateTime; +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java index 0c902a8f..53438d3c 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java @@ -1,5 +1,5 @@ package com.iqser.red.service.redaction.v1.model; public enum ChangeType { - ADDED, REMOVED + ADDED, REMOVED, CHANGED } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java index 2b7b506d..ff5a7329 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java @@ -5,6 +5,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; + @Data @Builder @AllArgsConstructor @@ -16,4 +18,8 @@ public class IdRemoval { private Status status; private boolean removeFromDictionary; -} \ No newline at end of file + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java index 3af345dd..c005e6d3 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java @@ -5,6 +5,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; + @Data @Builder @AllArgsConstructor @@ -16,4 +18,8 @@ public class ManualForceRedact { private Status status; private String legalBasis; -} \ No newline at end of file + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java index 7dc9120c..14924878 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java @@ -5,6 +5,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; + @Data @Builder @AllArgsConstructor @@ -15,7 +17,9 @@ public class ManualImageRecategorization { private String user; private Status status; private String type; - private String legalBasis; - private boolean redacted; -} \ No newline at end of file + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java index 4f0d211f..39210d07 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java @@ -5,6 +5,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; + @Data @Builder @AllArgsConstructor @@ -16,4 +18,8 @@ public class ManualLegalBasisChange { private Status status; private String legalBasis; -} \ No newline at end of file + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java index eb4fbecf..69eb03cf 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java @@ -5,6 +5,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; @@ -22,11 +23,12 @@ public class ManualRedactionEntry { private String legalBasis; private List positions = new ArrayList<>(); private Status status; + private boolean addToDictionary; - - private String section; - private int sectionNumber; - private boolean addToDossierDictionary; + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java deleted file mode 100644 index 74e385c0..00000000 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.iqser.red.service.redaction.v1.model; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.ArrayList; -import java.util.List; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class RedactionChangeLog { - - private List redactionLogEntry = new ArrayList<>(); - - private long dictionaryVersion = -1; - private long dossierDictionaryVersion = -1; - private long rulesVersion = -1; - private long legalBasisVersion = -1; - -} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java deleted file mode 100644 index a53d3b0e..00000000 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.iqser.red.service.redaction.v1.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.ArrayList; -import java.util.List; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class RedactionChangeLogEntry { - - private String id; - private String type; - private String value; - private String reason; - private int matchedRule; - private String legalBasis; - private boolean redacted; - private boolean isHint; - private boolean isRecommendation; - private String section; - private float[] color; - - @Builder.Default - private List positions = new ArrayList<>(); - private int sectionNumber; - private boolean manual; - private Status status; - private ManualRedactionType manualRedactionType; - private boolean isDictionaryEntry; - - private String textBefore; - private String textAfter; - - @Builder.Default - private List comments = new ArrayList<>(); - - private ChangeType changeType; - - private boolean isDossierDictionaryEntry; - - private boolean excluded; - -} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index 71a8413e..942e36a6 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -10,6 +10,13 @@ import java.util.List; @AllArgsConstructor public class RedactionLog { + + /** + * Version 0 Redaction Logs have manual redactions merged inside them + * Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime. + */ + private long analysisVersion; + private List redactionLogEntry; private List legalBasis; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java new file mode 100644 index 00000000..77fe374d --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class RedactionLogChanges { + + private RedactionLog redactionLog; + private boolean hasChanges; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index e347f0b3..dc14eebc 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -9,6 +9,8 @@ import lombok.NoArgsConstructor; import java.util.ArrayList; import java.util.List; + + @Data @Builder @NoArgsConstructor @@ -34,6 +36,7 @@ public class RedactionLogEntry { private boolean manual; private Status status; private ManualRedactionType manualRedactionType; + private String manualRedactionUserId; private boolean isDictionaryEntry; private String textBefore; @@ -46,6 +49,7 @@ public class RedactionLogEntry { private int endOffset; private boolean isImage; + private boolean imageHasTransparency; private boolean isDossierDictionaryEntry; @@ -54,4 +58,8 @@ public class RedactionLogEntry { private String recategorizationType; private String legalBasisChangeValue; + @EqualsAndHashCode.Exclude + @Builder.Default + private List changes = new ArrayList<>(); + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java index 263e7692..d92f0edf 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java @@ -1,5 +1,8 @@ package com.iqser.red.service.redaction.v1.model; +import java.util.HashSet; +import java.util.Set; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -15,4 +18,6 @@ public class RedactionRequest { private String fileId; private String dossierTemplateId; private ManualRedactions manualRedactions; + @Builder.Default + private Set excludedPages = new HashSet<>(); } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java index 38031b36..3a21bb47 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java @@ -30,4 +30,5 @@ public class SectionRectangle { private int numberOfParts; private List tableCells; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index 0a6037a6..d6bb66f9 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -8,10 +8,6 @@ import org.springframework.web.bind.annotation.RequestBody; public interface RedactionResource { - String RULE_SET_PARAMETER_NAME = "dossierTemplateId"; - String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}"; - - @PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest); @@ -24,13 +20,10 @@ public interface RedactionResource { @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest); - @PostMapping(value = "/rules/update" + RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE) - void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId); - @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE) void testRules(@RequestBody String rules); @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE) - RedactionLog getRedactionLogPreview(@RequestBody RedactionRequest redactionRequest); + RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java new file mode 100644 index 00000000..589e6dc3 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.server.client; + +import java.util.List; +import java.util.Map; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PostMapping; + +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecogintionEntity; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest; + +@FeignClient(name = "EntityRecognitionClient", url = "${entity-recognition-service.url}") +public interface EntityRecognitionClient { + + @PostMapping(value = "/find_authors", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + Map>> findAuthors(EntityRecognitionRequest entityRecognitionRequest); +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java new file mode 100644 index 00000000..b86a1b66 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecogintionEntity { + + private String value; + private int startOffset; + private int endOffset; + private String type; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java new file mode 100644 index 00000000..7e15bc69 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecognitionRequest { + + private List data; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResponse.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResponse.java new file mode 100644 index 00000000..af0d6beb --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResponse.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecognitionResponse { + + @Builder.Default + private Map> result = new HashMap<>(); + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java new file mode 100644 index 00000000..cb894b5d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecognitionResult { + + @Builder.Default + private Map> entities = new HashMap<>(); +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionSection.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionSection.java new file mode 100644 index 00000000..3172aa52 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionSection.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecognitionSection { + + private int sectionNumber; + private String text; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 0149d6a0..2ea691d7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -1,11 +1,7 @@ package com.iqser.red.service.redaction.v1.server.controller; import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.AnnotateRequest; -import com.iqser.red.service.redaction.v1.model.AnnotateResponse; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionRequest; -import com.iqser.red.service.redaction.v1.model.RedactionResult; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.resources.RedactionResource; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; @@ -13,8 +9,9 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; -import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; +import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; @@ -41,7 +38,7 @@ public class RedactionController implements RedactionResource { private final AnnotationService annotationService; private final PdfSegmentationService pdfSegmentationService; private final RedactionStorageService redactionStorageService; - private final RedactionLogCreatorService redactionLogCreatorService; + private final RedactionLogMergeService redactionLogMergeService; public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { @@ -143,14 +140,6 @@ public class RedactionController implements RedactionResource { } - - @Override - public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId) { - - droolsExecutionService.updateRules(dossierTemplateId); - } - - @Override public void testRules(@RequestBody String rules) { @@ -158,14 +147,22 @@ public class RedactionController implements RedactionResource { } @Override - public RedactionLog getRedactionLogPreview(RedactionRequest redactionRequest) { + public RedactionLog getRedactionLog(RedactionRequest redactionRequest) { + + log.info("Requested preview for: {}", redactionRequest); + dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId()); var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); - return redactionLogCreatorService.getRedactionLogPreview(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); + log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion()); + if (redactionLog.getAnalysisVersion() == 0) { + // old redaction logs are returned directly + return redactionLog; + } else { + return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages()); + } } - private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException { try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java index 45bcef6a..d3f6f2a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java @@ -189,18 +189,16 @@ public class PDFLinesTextStripper extends PDFTextStripper { COSName objectName = (COSName) arguments.get(0); PDXObject xobject = getResources().getXObject(objectName); if (xobject instanceof PDImageXObject) { - PDImageXObject pdfImage = (PDImageXObject) xobject; + PDImageXObject image = (PDImageXObject)xobject; + Matrix ctmNew = getGraphicsState().getCurrentTransformationMatrix(); - Rectangle2D imageBounds = calculateImagePosition(pdfImage); - - Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds - .getWidth(), (float) imageBounds.getHeight()); + Rectangle2D rect = new Rectangle2D.Float(ctmNew.getTranslateX(), ctmNew.getTranslateY(), ctmNew.getScaleX(), ctmNew.getScaleY()); // Memory Hack - sofReference kills me - FieldUtils.writeField(pdfImage, "cachedImageSubsampling", -1, true); + FieldUtils.writeField(image, "cachedImageSubsampling", -1, true); if (rect.getHeight() > 2 && rect.getWidth() > 2) { - this.images.add(new PdfImage(pdfImage.getImage(), rect, pageNumber)); + this.images.add(new PdfImage(image.getImage(), rect, pageNumber, image.getImage().getColorModel().hasAlpha())); } } } catch (Exception e) { @@ -209,21 +207,6 @@ public class PDFLinesTextStripper extends PDFTextStripper { } - private Rectangle2D calculateImagePosition(PDImageXObject pdfImage) throws IOException { - - Matrix ctm = getGraphicsState().getCurrentTransformationMatrix(); - - Rectangle2D imageBounds = pdfImage.getImage().getRaster().getBounds(); - - AffineTransform imageTransform = new AffineTransform(ctm.createAffineTransform()); - imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight()); - imageTransform.translate(0, -pdfImage.getHeight()); - - AffineTransform pageTransform = new AffineTransform(); - pageTransform.concatenate(imageTransform); - - return pageTransform.createTransformedShape(imageBounds).getBounds2D(); - } private float floatValue(COSBase value) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java index 5aab9c7a..63d382d5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java @@ -20,5 +20,6 @@ public class Image implements ReasonHolder { private int sectionNumber; private String section; private int page; + private boolean hasTransparency; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java index 1631717f..dbb390c0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java @@ -18,14 +18,16 @@ public class PdfImage { private RedRectangle2D position; private ImageType imageType; private boolean isAppendedToParagraph; + private boolean hasTransparency; @NonNull private int page; - public PdfImage(BufferedImage image, Rectangle2D position, int page) { + public PdfImage(BufferedImage image, Rectangle2D position, int page, boolean hasTransparency) { this.image = image; this.position = new RedRectangle2D(position.getX(), position.getY(), position.getWidth(), position.getHeight()); this.page = page; + this.hasTransparency = hasTransparency; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index dc878a20..17209efb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -72,6 +72,19 @@ public class Section { } + public boolean fileAttributeByIdEqualsIgnoreCase(String id, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + } + + public boolean fileAttributeByPlaceholderEqualsIgnoreCase(String placeholder, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + } + + public boolean fileAttributeByLabelEqualsIgnoreCase(String label, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + } + + public boolean rowEquals(String headerName, String value) { String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index ff772521..63fbadf6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -1,56 +1,30 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLog; import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; + +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @Service +@RequiredArgsConstructor public class AnalyzeResponseService { + private final RedactionServiceSettings redactionServiceSettings; + public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, - RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { + RedactionLog redactionLog, boolean hasUpdates) { - boolean hasHints = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isHint() && !entry.getType().equals("false_positive")); - - boolean hasRequests = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isManual() && entry.getStatus() - .equals(com.iqser.red.service.redaction.v1.model.Status.REQUESTED)); - - boolean hasRedactions = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isRedacted() && !entry.isManual() || entry.isManual() && entry.getStatus() - .equals(com.iqser.red.service.redaction.v1.model.Status.APPROVED)); - - boolean hasImages = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isHint() && entry.getType().equals("image") || entry.isImage()); - - boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog - .getRedactionLogEntry() - .isEmpty() && redactionChangeLog.getRedactionLogEntry() - .stream() - .anyMatch(entry -> !entry.getType().equals("false_positive")); return AnalyzeResult.builder() .dossierId(dossierId) .fileId(fileId) .duration(duration) .numberOfPages(pageCount) - .hasHints(hasHints) - .hasRedactions(hasRedactions) - .hasRequests(hasRequests) - .hasImages(hasImages) .hasUpdates(hasUpdates) + .analysisVersion(redactionServiceSettings.getAnalysisVersion()) .rulesVersion(redactionLog.getRulesVersion()) .dictionaryVersion(redactionLog.getDictionaryVersion()) .legalBasisVersion(redactionLog.getLegalBasisVersion()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index 0518eb3f..72d0d387 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -33,6 +33,7 @@ public class DictionaryService { public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) { + log.info("Updating dictionary data for: {} / {}", dossierTemplateId, dossierId); long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER); var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId); if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) { @@ -163,6 +164,7 @@ public class DictionaryService { public float[] getColor(String type, String dossierTemplateId) { + log.info("requested : {} / {}",type,dossierTemplateId); DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type); if (model != null) { return model.getColor(); @@ -210,13 +212,13 @@ public class DictionaryService { dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); } - return new Dictionary(copy, DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); + return new Dictionary(copy.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); } public float[] getRequestRemoveColor(String dossierTemplateId) { - return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor(); + return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestRemoveColor(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index bd26cc12..0ac53bc6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -63,7 +63,7 @@ public class DroolsExecutionService { rulesVersion = -1L; } - if (version > rulesVersion.longValue()) { + if (version > rulesVersion) { rulesVersionPerDossierTemplateId.put(dossierTemplateId, version); return createOrUpdateKieContainer(dossierTemplateId); } @@ -128,7 +128,7 @@ public class DroolsExecutionService { if (rulesVersion == null) { return -1; } - return rulesVersion.longValue(); + return rulesVersion; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index e6d00c15..67e86ca0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,19 +1,23 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.redaction.v1.model.FileAttribute; -import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; -import com.iqser.red.service.redaction.v1.model.ManualRedactions; -import com.iqser.red.service.redaction.v1.model.Point; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.SectionArea; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.*; +import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecogintionEntity; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionResponse; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; + import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.kie.api.runtime.KieContainer; @@ -32,9 +36,12 @@ public class EntityRedactionService { private final DictionaryService dictionaryService; private final DroolsExecutionService droolsExecutionService; private final SurroundingWordsService surroundingWordsService; + private final EntityRecognitionClient entityRecognitionClient; + private final RedactionServiceSettings redactionServiceSettings; - public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId, List fileAttributes) { + public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, + String dossierId, List fileAttributes) { dictionaryService.updateDictionary(dossierTemplateId, dossierId); KieContainer container = droolsExecutionService.updateRules(dossierTemplateId); @@ -46,35 +53,13 @@ public class EntityRedactionService { if (dictionary.hasLocalEntries()) { - Map> hintsPerSectionNumber = new HashMap<>(); - documentEntities.stream().forEach(entity -> { - if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) { - hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()) - .add(entity); - } - }); - + Map> hintsPerSectionNumber = getHintsPerSection(documentEntities, dictionary); Set foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber, fileAttributes); EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary); EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities); } - for (Entity entity : documentEntities) { - Map> sequenceOnPage = new HashMap<>(); - for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { - sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()) - .add(entityPositionSequence); - } - - for (Map.Entry> entry : sequenceOnPage.entrySet()) { - classifiedDoc.getEntities() - .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) - .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry - .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity - .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity - .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry())); - } - } + classifiedDoc.setEntities(convertToEnititesPerPage(documentEntities)); dictionaryService.updateExternalDictionary(dictionary, dossierTemplateId); @@ -83,9 +68,44 @@ public class EntityRedactionService { } + public Map> convertToEnititesPerPage(Set entities) { + + Map> entitiesPerPage = new HashMap<>(); + for (Entity entity : entities) { + Map> sequenceOnPage = new HashMap<>(); + for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { + sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()) + .add(entityPositionSequence); + } + + for (Map.Entry> entry : sequenceOnPage.entrySet()) { + entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) + .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry + .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity + .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity + .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry())); + } + } + return entitiesPerPage; + } + + + public Map> getHintsPerSection(Set entities, Dictionary dictionary) { + + Map> hintsPerSectionNumber = new HashMap<>(); + entities.stream().forEach(entity -> { + if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) { + hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity); + } + }); + return hintsPerSectionNumber; + } + + private Set findEntities(Document classifiedDoc, KieContainer kieContainer, ManualRedactions manualRedactions, Dictionary dictionary, boolean local, - Map> hintsPerSectionNumber, List fileAttributes) { + Map> hintsPerSectionNumber, + List fileAttributes) { Set documentEntities = new HashSet<>(); @@ -96,9 +116,9 @@ public class EntityRedactionService { List tables = paragraph.getTables(); for (Table table : tables) { if (table.getColCount() == 2) { - sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); + sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); } else { - sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); + sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); } sectionNumber.incrementAndGet(); } @@ -125,47 +145,52 @@ public class EntityRedactionService { } sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { - Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); - documentEntities.addAll(analysedRowSection.getEntities()); + Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); + documentEntities.addAll(analysedSection.getEntities()); - for (Image image : analysedRowSection.getImages()) { + for (Image image : analysedSection.getImages()) { classifiedDoc.getImages().computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image); } - analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> { - if (dictionary.isRecommendation(key)) { - analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> { - if (!dictionary.containsValue(key, value)) { - dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); - } - }); - } else { - analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> { - - if (dictionary.getLocalAccessMap().get(key) == null) { - log.warn("Dictionary {} is null", key); - } - - if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) { - log.warn("Dictionary {} localEntries is null", key); - } - - dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); - }); - } - }); - + addLocalValuesToDictionary(analysedSection, dictionary); }); return documentEntities; } + public void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) { + + analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> { + if (dictionary.isRecommendation(key)) { + analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> { + if (!dictionary.containsValue(key, value)) { + dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); + } + }); + } else { + analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> { + + if (dictionary.getLocalAccessMap().get(key) == null) { + log.warn("Dictionary {} is null", key); + } + + if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) { + log.warn("Dictionary {} localEntries is null", key); + } + + dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); + }); + } + }); + } + + private List processTablePerRow(Document classifiedDoc, Table table, - ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, - Map> hintsPerSectionNumber, List fileAttributes) { + Map> hintsPerSectionNumber, + List fileAttributes) { List sectionSearchableTextPairs = new ArrayList<>(); @@ -190,7 +215,6 @@ public class EntityRedactionService { sectionText.getSectionAreas().add(sectionArea); sectionText.getTextBlocks().addAll(cell.getTextBlocks()); - addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue()); int cellStart = start; if (!cell.isHeaderCell()) { @@ -251,7 +275,6 @@ public class EntityRedactionService { private List processTableAsOneText(Document classifiedDoc, Table table, - ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber, @@ -280,7 +303,6 @@ public class EntityRedactionService { for (TextBlock textBlock : cell.getTextBlocks()) { entireTableText.addAll(textBlock.getSequences()); } - addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue()); } } @@ -334,13 +356,12 @@ public class EntityRedactionService { sectionText.setSectionNumber(sectionNumber.intValue()); sectionText.setTable(false); sectionText.setImages(images.stream() - .map(image -> convert(image, sectionNumber.intValue(), headline)) + .map(image -> convertAndRecategorize(image, sectionNumber.intValue(), headline, manualRedactions)) .collect(Collectors.toSet())); sectionText.setTextBlocks(paragraphTextBlocks); classifiedDoc.getSectionText().add(sectionText); } - addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue()); Set entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local); surroundingWordsService.addSurroundingText(entities, searchableText, dictionary); @@ -357,7 +378,7 @@ public class EntityRedactionService { .searchableText(searchableText) .dictionary(dictionary) .images(images.stream() - .map(image -> convert(image, sectionNumber.intValue(), headline)) + .map(image -> convertAndRecategorize(image, sectionNumber.intValue(), headline, manualRedactions)) .collect(Collectors.toSet())) .fileAttributes(fileAttributes) .build(), searchableText); @@ -376,39 +397,26 @@ public class EntityRedactionService { String lowercaseInputString = searchableString.toLowerCase(); for (DictionaryModel model : dictionary.getDictionaryModels()) { if (model.isCaseInsensitive()) { - found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary())); + found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model + .isDossierDictionary())); } else { - found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary())); + found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model + .isDossierDictionary())); } } + if (redactionServiceSettings.isEnableEntityRecognition() && !local) { + found.addAll(getAiEntities(sectionNumber, searchableString, headline)); + } + return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); } - private void addSectionToManualRedactions(List textBlocks, ManualRedactions manualRedactions, - String section, int sectionNumber) { + private Image convertAndRecategorize(PdfImage pdfImage, int sectionNumber, String headline, + ManualRedactions manualRedactions) { - if (manualRedactions == null || manualRedactions.getEntriesToAdd().isEmpty()) { - return; - } - - for (TextBlock textBlock : textBlocks) { - for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) { - for (Rectangle rectangle : manualRedactionEntry.getPositions()) { - if (textBlock.contains(rectangle)) { - manualRedactionEntry.setSection(section); - manualRedactionEntry.setSectionNumber(sectionNumber); - } - } - } - } - } - - - private Image convert(PdfImage pdfImage, int sectionNumber, String headline) { - - return Image.builder() + Image image = Image.builder() .type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType() .name() .toLowerCase(Locale.ROOT)) @@ -416,7 +424,44 @@ public class EntityRedactionService { .sectionNumber(sectionNumber) .section(headline) .page(pdfImage.getPage()) + .hasTransparency(pdfImage.isHasTransparency()) .build(); + + String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); + if (manualRedactions != null && manualRedactions.getImageRecategorizations() != null) { + for (ManualImageRecategorization imageRecategorization : manualRedactions.getImageRecategorizations()) { + if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId() + .equals(imageId)) { + image.setType(imageRecategorization.getType()); + } + } + } + + return image; + } + + + private Set getAiEntities(int sectionNumber, String searchableString, String headline) { + + Set found = new HashSet<>(); + + Map>> response = entityRecognitionClient.findAuthors(EntityRecognitionRequest + .builder() + .data(List.of(EntityRecognitionSection.builder() + .sectionNumber(sectionNumber) + .text(searchableString) + .build())) + .build()); + + EntityRecognitionResponse entityRecognitionResponse = new EntityRecognitionResponse(response.get("result:")); + + if (entityRecognitionResponse.getResult() != null && entityRecognitionResponse.getResult() + .containsKey(String.valueOf(sectionNumber))) { + entityRecognitionResponse.getResult().get(String.valueOf(sectionNumber)).forEach(res -> { + found.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false)); + }); + } + return found; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index ddc67900..1daf766b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -1,54 +1,32 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.RequestBody; - import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; -import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import com.iqser.red.service.redaction.v1.model.Comment; -import com.iqser.red.service.redaction.v1.model.IdRemoval; -import com.iqser.red.service.redaction.v1.model.ManualForceRedact; -import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; -import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange; -import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; -import com.iqser.red.service.redaction.v1.model.ManualRedactions; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.model.SectionArea; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; -import com.iqser.red.service.redaction.v1.server.redaction.model.Section; -import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; +import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.kie.api.runtime.KieContainer; +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.RequestBody; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + @Slf4j @Service @RequiredArgsConstructor @@ -64,6 +42,7 @@ public class ReanalyzeService { private final RedactionChangeLogService redactionChangeLogService; private final AnalyzeResponseService analyzeResponseService; private final LegalBasisClient legalBasisClient; + private final RedactionServiceSettings redactionServiceSettings; public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { @@ -85,13 +64,12 @@ public class ReanalyzeService { entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest .getDossierId(), analyzeRequest.getFileAttributes()); - redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest - .getDossierTemplateId()); + redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getDossierTemplateId()); log.info("Redaction analysis successful..."); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() + var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() .getDossierTemplateVersion(), classifiedDoc.getDictionaryVersion() .getDossierVersion(), classifiedDoc.getRulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); @@ -100,9 +78,9 @@ public class ReanalyzeService { log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc .getDictionaryVersion(), analyzeRequest.getDossierTemplateId()); - // first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data - var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); - // store redactionLog + var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); + redactionLog = redactionLogChange.getRedactionLog(); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc .getSectionText())); @@ -110,7 +88,7 @@ public class ReanalyzeService { .getSectionGrid()); long duration = System.currentTimeMillis() - startTime; - return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog); + return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, redactionLogChange.isHasChanges()); } @@ -130,20 +108,66 @@ public class ReanalyzeService { DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions()); - Map> comments = null; - Set manualAdds = null; + Set sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse().isEmpty() ? analyzeRequest.getSectionsToReanalyse() : + findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest); - if (analyzeRequest.getManualRedactions() != null) { - // TODO comments will be removed from redactionLog, so we ignore this first. - comments = analyzeRequest.getManualRedactions().getComments(); - manualAdds = analyzeRequest.getManualRedactions().getEntriesToAdd(); + if (sectionsToReanalyse.isEmpty()) { + return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); } + List reanalysisSections = text.getSectionTexts() + .stream() + .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber())) + .collect(Collectors.toList()); + + KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); + + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest + .getDossierId()); + + Map> imagesPerPage = new HashMap<>(); + Set entities = findEntities(reanalysisSections, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage); + + if (dictionary.hasLocalEntries()) { + Map> hintsPerSectionNumber = entityRedactionService.getHintsPerSection(entities, dictionary); + Set foundByLocal = findEntities(reanalysisSections, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage); + EntitySearchUtils.addEntitiesWithHigherRank(entities, foundByLocal, dictionary); + EntitySearchUtils.removeEntitiesContainedInLarger(entities); + } + + Map> entitiesPerPage = entityRedactionService.convertToEnititesPerPage(entities); + + List newRedactionLogEntries = new ArrayList<>(); + for (int page = 1; page <= text.getNumberOfPages(); page++) { + if (entitiesPerPage.get(page) != null) { + newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, page, analyzeRequest + .getDossierTemplateId())); + } + + if (imagesPerPage.get(page) != null) { + newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, page, analyzeRequest + .getDossierTemplateId())); + } + + } + + redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); + redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); + AnalyzeResult analyzeResult = finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); + analyzeResult.setWasReanalyzed(true); + return analyzeResult; + } + + + private Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, + Text text, AnalyzeRequest analyzeRequest) { + + Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); + Set sectionsToReanalyse = new HashSet<>(); Map> imageEntries = new HashMap<>(); for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) { + if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { sectionsToReanalyse.add(entry.getSectionNumber()); } if (entry.isImage() || entry.getType().equals("image")) { @@ -157,47 +181,24 @@ public class ReanalyzeService { sectionsToReanalyse.add(sectionText.getSectionNumber()); } - if (manualAdds != null) { - for (SectionArea sectionArea : sectionText.getSectionAreas()) { - for (ManualRedactionEntry manualAdd : manualAdds) { - for (Rectangle manualPosition : manualAdd.getPositions()) { - if (sectionArea.contains(manualPosition)) { - manualAdd.setSection(sectionText.getHeadline()); - manualAdd.setSectionNumber(sectionText.getSectionNumber()); - } - } - } - } - } } log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest); - if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) { - return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); - } + return sectionsToReanalyse; + } - List reanalysisSections = new ArrayList<>(); - for (SectionText sectionText : text.getSectionTexts()) { - - if (sectionsToReanalyse.contains(sectionText.getSectionNumber())) { - reanalysisSections.add(sectionText); - } - } - - //-- - - KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest - .getDossierId()); + private Set findEntities(List reanalysisSections, Dictionary dictionary, + KieContainer kieContainer, AnalyzeRequest analyzeRequest, boolean local, + Map> hintsPerSectionNumber, + Map> imagesPerPage) { List sectionSearchableTextPairs = new ArrayList<>(); for (SectionText reanalysisSection : reanalysisSections) { Set entities = entityRedactionService.findEntities(reanalysisSection.getSearchableText(), reanalysisSection - .getHeadline(), reanalysisSection.getSectionNumber(), dictionary, false); + .getHeadline(), reanalysisSection.getSectionNumber(), dictionary, local); if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) { surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection .getCellStarts()); @@ -205,10 +206,28 @@ public class ReanalyzeService { surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary); } + if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages() + .isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions() + .getImageRecategorizations() != null) { + for (Image image : reanalysisSection.getImages()) { + String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); + for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() + .getImageRecategorizations()) { + if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId() + .equals(imageId)) { + image.setType(imageRecategorization.getType()); + } + } + } + } + sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() .isLocal(false) .dictionaryTypes(dictionary.getTypes()) - .entities(entities) + .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream + .concat(entities.stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()) + .stream()) + .collect(Collectors.toSet()) : entities) .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks()) .searchText(reanalysisSection.getSearchableText().toString()) .headline(reanalysisSection.getHeadline()) @@ -222,56 +241,19 @@ public class ReanalyzeService { } Set entities = new HashSet<>(); - Map> imagesPerPage = new HashMap<>(); sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { - Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); - entities.addAll(analysedRowSection.getEntities()); + Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); + entities.addAll(analysedSection.getEntities()); EntitySearchUtils.removeEntitiesContainedInLarger(entities); - for (Image image : analysedRowSection.getImages()) { + for (Image image : analysedSection.getImages()) { imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image); } + entityRedactionService.addLocalValuesToDictionary(analysedSection, dictionary); }); - Map> entitiesPerPage = new HashMap<>(); - for (Entity entity : entities) { - Map> sequenceOnPage = new HashMap<>(); - for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { - sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()) - .add(entityPositionSequence); - } - - for (Map.Entry> entry : sequenceOnPage.entrySet()) { - entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) - .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry - .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity - .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity - .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry())); - } - } - - List newRedactionLogEntries = new ArrayList<>(); - for (int page = 1; page <= text.getNumberOfPages(); page++) { - if (entitiesPerPage.get(page) != null) { - newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest - .getDossierTemplateId())); - } - - if (imagesPerPage.get(page) != null) { - newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest - .getDossierTemplateId())); - } - - newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest - .getDossierTemplateId())); - } - - redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); - redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); - AnalyzeResult analyzeResult = finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); - analyzeResult.setWasReanalyzed(true); - return analyzeResult; + return entities; } @@ -284,25 +266,29 @@ public class ReanalyzeService { excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); - var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); + var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); long duration = System.currentTimeMillis() - startTime; return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text - .getNumberOfPages(), redactionLog, changeLog); + .getNumberOfPages(), redactionLogChange.getRedactionLog(), redactionLogChange.isHasChanges()); } - private Set getForceAndRemoveIds(ManualRedactions manualRedactions) { + private Set getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) { if (manualRedactions == null) { return new HashSet<>(); } - return Stream.concat(manualRedactions.getManualLegalBasisChanges().stream().map(ManualLegalBasisChange::getId), - Stream.concat(manualRedactions.getImageRecategorizations().stream().map(ManualImageRecategorization::getId), - Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)))) + return Stream.concat(manualRedactions.getManualLegalBasisChanges() + .stream() + .map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorizations() + .stream() + .map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove() + .stream() + .map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)))) .collect(Collectors.toSet()); } @@ -318,21 +304,18 @@ public class ReanalyzeService { .sectionNumber(entry.getSectionNumber()) .section(entry.getSection()) .page(position.getPage()) + .hasTransparency(entry.isImageHasTransparency()) .build(); } private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { - redactionLog.getRedactionLogEntry().forEach(entry -> { - entry.getPositions().forEach(pos -> { - if (excludedPages != null && excludedPages.contains(pos.getPage())) { - entry.setExcluded(true); - } else { - entry.setExcluded(false); - } - }); - }); + if(excludedPages != null && !excludedPages.isEmpty()) { + redactionLog.getRedactionLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { if (excludedPages.contains(pos.getPage())) { + entry.setExcluded(true); + }})); + } } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index 1cc743e5..41772291 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -1,19 +1,25 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.ChangeType; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLog; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLogEntry; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; +import com.iqser.red.service.redaction.v1.model.Change; +import com.iqser.red.service.redaction.v1.model.ChangeType; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogChanges; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @@ -22,76 +28,77 @@ public class RedactionChangeLogService { private final RedactionStorageService redactionStorageService; - public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) { - try { - RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId); - var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog); - redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog); - return changeLog; - } catch (Exception e) { - log.debug("Previous redaction log not available"); - return null; - } - - } - - - private RedactionChangeLog createChangeLog(RedactionLog currentRedactionLog, RedactionLog previousRedactionLog) { + public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) { + RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId); if (previousRedactionLog == null) { - return null; + currentRedactionLog.getRedactionLogEntry().forEach(entry -> { + entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + }); + return new RedactionLogChanges(currentRedactionLog, false); } - List added = new ArrayList<>(currentRedactionLog.getRedactionLogEntry()); - added.removeAll(previousRedactionLog.getRedactionLogEntry()); + List notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> !entry.getChanges() + .get(entry.getChanges().size() - 1) + .getType() + .equals(ChangeType.REMOVED)) + .collect(Collectors.toList()); - List removed = new ArrayList<>(previousRedactionLog.getRedactionLogEntry()); + Set added = new HashSet<>(currentRedactionLog.getRedactionLogEntry()); + added.removeAll(notRemovedPreviousEntries); + + Set removed = new HashSet<>(notRemovedPreviousEntries); removed.removeAll(currentRedactionLog.getRedactionLogEntry()); - List changeLogEntries = added.stream() - .map(entry -> convert(entry, ChangeType.ADDED)) - .collect(Collectors.toList()); - changeLogEntries.addAll(removed.stream() - .map(entry -> convert(entry, ChangeType.REMOVED)) - .collect(Collectors.toList())); + Map addedIds = new HashMap<>(); + added.forEach(entry -> { + addedIds.put(entry.getId(), entry); + }); - return new RedactionChangeLog(changeLogEntries, - currentRedactionLog.getDictionaryVersion(), - currentRedactionLog.getDossierDictionaryVersion(), - currentRedactionLog.getRulesVersion(), - currentRedactionLog.getLegalBasisVersion()); - } + Set removedIds = new HashSet<>(); + removed.forEach(entry -> { + removedIds.add(entry.getId()); + }); + List newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry(); - private RedactionChangeLogEntry convert(RedactionLogEntry entry, ChangeType changeType) { + List toRemove = new ArrayList<>(); + newRedactionLogEntries.forEach(entry -> { + if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now())); + var newEntry = addedIds.get(entry.getId()); + newEntry.setChanges(changes); + addedIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } else if (removedIds.contains(entry.getId())) { + entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now())); + } else if (addedIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + var newEntry = addedIds.get(entry.getId()); + newEntry.setChanges(changes); + addedIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } + }); - return RedactionChangeLogEntry.builder() - .id(entry.getId()) - .type(entry.getType()) - .value(entry.getValue()) - .reason(entry.getReason()) - .matchedRule(entry.getMatchedRule()) - .legalBasis(entry.getLegalBasis()) - .redacted(entry.isRedacted()) - .isHint(entry.isHint()) - .isRecommendation(entry.isRecommendation()) - .section(entry.getSection()) - .color(entry.getColor()) - .positions(entry.getPositions()) - .sectionNumber(entry.getSectionNumber()) - .manual(entry.isManual()) - .status(entry.getStatus()) - .manualRedactionType(entry.getManualRedactionType()) - .isDictionaryEntry(entry.isDictionaryEntry()) - .textBefore(entry.getTextBefore()) - .textAfter(entry.getTextAfter()) - .comments(entry.getComments()) - .changeType(changeType) - .isDossierDictionaryEntry(entry.isDossierDictionaryEntry()) - .excluded(entry.isExcluded()) - .build(); + newRedactionLogEntries.removeAll(toRemove); + + addedIds.forEach((k, v) -> { + if(v.getChanges().isEmpty()) { + v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + } + newRedactionLogEntries.add(v); + }); + + currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries); + + return new RedactionLogChanges(currentRedactionLog, !addedIds.isEmpty() || !removedIds.isEmpty()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 5c5e1e70..ec1f78db 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -1,6 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.model.CellRectangle; +import com.iqser.red.service.redaction.v1.model.Point; +import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.SectionRectangle; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; @@ -9,17 +13,19 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.ReasonHolder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import lombok.RequiredArgsConstructor; import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; @Service @@ -29,10 +35,7 @@ public class RedactionLogCreatorService { private final DictionaryService dictionaryService; - public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions, - String dossierTemplateId) { - - Set manualRedactionPages = getManualRedactionPages(manualRedactions); + public void createRedactionLog(Document classifiedDoc, int numberOfPages, String dossierTemplateId) { for (int page = 1; page <= numberOfPages; page++) { @@ -40,24 +43,18 @@ public class RedactionLogCreatorService { if (classifiedDoc.getEntities().get(page) != null) { classifiedDoc.getRedactionLogEntities() - .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId)); - } - - if (manualRedactionPages.contains(page)) { - classifiedDoc.getRedactionLogEntities() - .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId)); + .addAll(addEntries(classifiedDoc.getEntities(), page, dossierTemplateId)); } if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) { classifiedDoc.getRedactionLogEntities() - .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId)); + .addAll(addImageEntries(classifiedDoc.getImages(), page, dossierTemplateId)); } } } - public List addImageEntries(Map> images, ManualRedactions manualRedactions, - int pageNumber, String dossierTemplateId) { + public List addImageEntries(Map> images, int pageNumber, String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -67,7 +64,7 @@ public class RedactionLogCreatorService { RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() .id(id) - .color(getColorForImage(image.getType(), dossierTemplateId, false, image.isRedaction())) + .color(getColor(image.getType(), dossierTemplateId, image.isRedaction())) .isImage(true) .type(image.getType()) .redacted(image.isRedaction()) @@ -83,9 +80,9 @@ public class RedactionLogCreatorService { .getWidth(), (float) image.getPosition().getHeight(), pageNumber))) .sectionNumber(image.getSectionNumber()) .section(image.getSection()) + .imageHasTransparency(image.isHasTransparency()) .build(); - processImageEntry(manualRedactions, dossierTemplateId, image, redactionLogEntry); redactionLogEntities.add(redactionLogEntry); } @@ -93,138 +90,8 @@ public class RedactionLogCreatorService { return redactionLogEntities; } - private void processImageEntry(ManualRedactions manualRedactions, String dossierTemplateId, ReasonHolder image, RedactionLogEntry redactionLogEntry) { - if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { - for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { - if (recategorization.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (recategorization.getStatus().equals(Status.APPROVED)) { - image.setRedaction(recategorization.isRedacted()); - redactionLogEntry.setType(recategorization.getType()); - redactionLogEntry.setHint(dictionaryService.isHint(recategorization.getType(), dossierTemplateId)); - redactionLogEntry.setRedacted(recategorization.isRedacted()); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setLegalBasis(recategorization.getLegalBasis()); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", recategorized by manual override"); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - } else if (recategorization.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to recategorize"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setRecategorizationType(recategorization.getType()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); - } - } - } - - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - image.setRedaction(false); - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to remove"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - } - } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - image.setRedaction(true); - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", forced by manual override"); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to force redact"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - } - } - } - - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis was manually changed"); - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis change requested"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } - } - } - } - - - private Set getManualRedactionPages(ManualRedactions manualRedactions) { - - Set manualRedactionPages = new HashSet<>(); - - if (manualRedactions == null) { - return manualRedactionPages; - } - - manualRedactions.getEntriesToAdd().forEach(entry -> { - entry.getPositions().forEach(pos -> { - manualRedactionPages.add(pos.getPage()); - }); - }); - return manualRedactionPages; - } - - - public List addEntries(Map> entities, ManualRedactions manualRedactions, - int page, String dossierTemplateId) { + public List addEntries(Map> entities, int page, String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -247,7 +114,6 @@ public class RedactionLogCreatorService { } redactionLogEntry.setId(entityPositionSequence.getId()); - processRedactionLogEntry(manualRedactions, dossierTemplateId, redactionLogEntry, entity); if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) { List rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences() @@ -271,109 +137,6 @@ public class RedactionLogCreatorService { return redactionLogEntities; } - private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry, ReasonHolder reasonHolder) { - - List comments = null; - - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(redactionLogEntry.getId())) { - comments = manualRedactions.getComments().get(manualRemoval.getId()); - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - reasonHolder.setRedaction(false); - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to remove"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - } - } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - reasonHolder.setRedaction(true); - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", forced by manual override"); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to force redact"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - } - } - } - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis was manually changed"); - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis change requested"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } - } - } - - - if (manualRedactions != null) { - comments = manualRedactions.getComments().get(redactionLogEntry.getId()); - } - - redactionLogEntry.setComments(comments); - } - - private String mergeReasonIfNecessary(String currentReason, String addition) { - if (currentReason != null) { - if (!currentReason.contains(addition)) { - return currentReason + addition; - } - return currentReason; - } else { - return ""; - } - } - private List getRectanglesPerLine(List textPositions, int page) { @@ -402,73 +165,10 @@ public class RedactionLogCreatorService { } - public List addManualAddEntries(Set manualAdds, - Map> comments, int page, - String dossierTemplateId) { - - List redactionLogEntities = new ArrayList<>(); - - if (manualAdds == null) { - return redactionLogEntities; - } - - for (ManualRedactionEntry manualRedactionEntry : manualAdds) { - - String id = manualRedactionEntry.getId(); - - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId); - - List rectanglesOnPage = new ArrayList<>(); - for (Rectangle rectangle : manualRedactionEntry.getPositions()) { - if (page == rectangle.getPage()) { - rectanglesOnPage.add(rectangle); - redactionLogEntry.getPositions().add(rectangle); - } - } - - redactionLogEntry.setComments(comments.get(id)); - if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) { - redactionLogEntities.add(redactionLogEntry); - } - } - - return redactionLogEntities; - } - - - private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { - - return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary(); - } - - - private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, - String dossierTemplateId) { - - return RedactionLogEntry.builder() - .id(id) - .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) - .reason(manualRedactionEntry.getReason()) - .legalBasis(manualRedactionEntry.getLegalBasis()) - .value(manualRedactionEntry.getValue()) - .type(manualRedactionEntry.getType()) - .redacted(true) - .isHint(false) - .section(manualRedactionEntry.getSection()) - .sectionNumber(manualRedactionEntry.getSectionNumber()) - .manual(true) - .status(manualRedactionEntry.getStatus()) - .manualRedactionType(ManualRedactionType.ADD) - .isDictionaryEntry(false) - .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) - .build(); - } - - private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) { return RedactionLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, false, entity.isRedaction())) + .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction())) .reason(entity.getRedactionReason()) .legalBasis(entity.getLegalBasis()) .value(entity.getWord()) @@ -489,59 +189,6 @@ public class RedactionLogCreatorService { } - private float[] getColor(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) { - - if (requestedToRemove) { - return dictionaryService.getRequestRemoveColor(dossierTemplateId); - } - if (!isRedaction && !isHint(type, dossierTemplateId)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) { - - if (status.equals(Status.REQUESTED)) { - return dictionaryService.getRequestAddColor(dossierTemplateId); - } else if (status.equals(Status.DECLINED)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return getColor(type, dossierTemplateId); - } - - - private float[] getColor(String type, String dossierTemplateId) { - - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private float[] getColorForImage(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) { - - if (requestedToRemove) { - return dictionaryService.getRequestRemoveColor(dossierTemplateId); - } - if (!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private boolean isHint(String type, String dossierTemplateId) { - - return dictionaryService.isHint(type, dossierTemplateId); - } - - - private boolean isRecommendation(String type, String dossierTemplateId) { - - return dictionaryService.isRecommendation(type, dossierTemplateId); - } - - private void addSectionGrid(Document classifiedDoc, int page) { for (Paragraph paragraph : classifiedDoc.getParagraphs()) { @@ -585,74 +232,26 @@ public class RedactionLogCreatorService { } } - public RedactionLog getRedactionLogPreview(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { + private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { - var manualRedactionPages = getManualRedactionPages(manualRedactions); - - // generate all manual entries - var manualRedactionLogEntries = new HashMap(); - for (var page : manualRedactionPages) { - - var pageEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId); - - for (var entry : pageEntries) { - manualRedactionLogEntries.put(entry.getId(), entry); - } + if (!isRedaction && !isHint(type, dossierTemplateId)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); } - - for (var manualEntry : manualRedactionLogEntries.values()) { - var existingEntry = redactionLog.getRedactionLogEntry().stream().filter(e -> e.getId().equals(manualEntry.getId())).findAny(); - if (existingEntry.isPresent()) { - // if it has already been processed of sorts, update it - BeanUtils.copyProperties(manualEntry, existingEntry.get()); - } else { - // not yet in the redaction-log - add it - redactionLog.getRedactionLogEntry().add(manualEntry); - } - } - - - for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - - var reasonHolder = new PreviewReasonHolder(entry); - - if (entry.isImage()) { - processImageEntry(manualRedactions, dossierTemplateId, reasonHolder, entry); - } - - processRedactionLogEntry(manualRedactions, dossierTemplateId, entry, reasonHolder); - } - - return redactionLog; + return dictionaryService.getColor(type, dossierTemplateId); } - public static class PreviewReasonHolder implements ReasonHolder { - private final RedactionLogEntry entry; + private boolean isHint(String type, String dossierTemplateId) { - public PreviewReasonHolder(RedactionLogEntry entry) { - this.entry = entry; - } - - @Override - public String getRedactionReason() { - return entry.getReason(); - } - - @Override - public void setRedactionReason(String reason) { - entry.setReason(reason); - } - - @Override - public boolean isRedaction() { - return entry.isRedacted(); - } - - @Override - public void setRedaction(boolean value) { - entry.setRedacted(value); - } + return dictionaryService.isHint(type, dossierTemplateId); } + + + private boolean isRecommendation(String type, String dossierTemplateId) { + + return dictionaryService.isRecommendation(type, dossierTemplateId); + } + + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java new file mode 100644 index 00000000..5c1d9fcc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -0,0 +1,328 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.model.Comment; +import com.iqser.red.service.redaction.v1.model.IdRemoval; +import com.iqser.red.service.redaction.v1.model.ManualForceRedact; +import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; +import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange; +import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; +import com.iqser.red.service.redaction.v1.model.ManualRedactionType; +import com.iqser.red.service.redaction.v1.model.ManualRedactions; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.Status; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class RedactionLogMergeService { + + private final DictionaryService dictionaryService; + + + public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, + ManualRedactions manualRedactions, Set excludedPages) { + + log.info("Merging Redaction log with manual redactions {}", manualRedactions); + if (manualRedactions != null) { + + var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); + + redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); + + var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions); + + for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { + + processRedactionLogEntry(manualRedactionWrappers.stream() + .filter(mr -> entry.getId().equals(mr.getId())) + .collect(Collectors.toList()), dossierTemplateId, entry); + + entry.setComments(manualRedactions.getComments().get(entry.getId())); + + if (excludedPages != null && !excludedPages.isEmpty()) { + entry.getPositions().forEach(pos -> { + if (excludedPages.contains(pos.getPage())) { + entry.setExcluded(true); + } + }); + } + } + + } + + return redactionLog; + } + + + private List createManualRedactionWrappers(ManualRedactions manualRedactions) { + + List manualRedactionWrappers = new ArrayList<>(); + + manualRedactions.getImageRecategorizations().forEach(item -> { + if (item.getSoftDeletedTime() == null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); + } + }); + + manualRedactions.getIdsToRemove().forEach(item -> { + if (item.getSoftDeletedTime() == null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); + } + }); + + manualRedactions.getForceRedacts().forEach(item -> { + if (item.getSoftDeletedTime() == null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); + } + }); + + manualRedactions.getManualLegalBasisChanges().forEach(item -> { + if (item.getSoftDeletedTime() == null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); + } + }); + + Collections.sort(manualRedactionWrappers); + + return manualRedactionWrappers; + } + + + private void processRedactionLogEntry(List manualRedactionWrappers, + String dossierTemplateId, RedactionLogEntry redactionLogEntry) { + + manualRedactionWrappers.forEach(mrw -> { + + if (mrw.getItem() instanceof ManualImageRecategorization) { + var imageRecategorization = (ManualImageRecategorization) mrw.getItem(); + String manualOverrideReason = null; + if (imageRecategorization.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setType(imageRecategorization.getType()); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); + } else if (imageRecategorization.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry + .isRedacted(), false)); + redactionLogEntry.setRecategorizationType(imageRecategorization.getType()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); + } + + if (mrw.getItem() instanceof IdRemoval) { + var manualRemoval = (IdRemoval) mrw.getItem(); + String manualOverrideReason = null; + if (manualRemoval.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setRedacted(false); + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry + .isRedacted(), true)); + } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry + .isRedacted(), false)); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser()); + redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); + redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + } + + if (mrw.getItem() instanceof ManualForceRedact) { + var manualForceRedact = (ManualForceRedact) mrw.getItem(); + String manualOverrideReason = null; + if (manualForceRedact.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setRedacted(true); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry + .isRedacted(), false)); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry + .isRedacted(), false)); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + + } + + if (mrw.getItem() instanceof ManualLegalBasisChange) { + var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem(); + String manualOverrideReason = null; + if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); + redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry + .isRedacted(), false)); + redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + } + + }); + + } + + + private String mergeReasonIfNecessary(String currentReason, String addition) { + + if (currentReason != null) { + if (!currentReason.contains(addition)) { + return currentReason + addition; + } + return currentReason; + } else { + return ""; + } + } + + + public List addManualAddEntries(Set manualAdds, + Map> comments, String dossierTemplateId) { + + List redactionLogEntries = new ArrayList<>(); + + for (ManualRedactionEntry manualRedactionEntry : manualAdds) { + + if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) { + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry + .getId(), dossierTemplateId); + redactionLogEntry.setPositions(manualRedactionEntry.getPositions()); + redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId())); + redactionLogEntries.add(redactionLogEntry); + } + } + + return redactionLogEntries; + } + + + private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { + + return manualRedactionEntry.getStatus() + .equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); + } + + + private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, + String dossierTemplateId) { + + return RedactionLogEntry.builder() + .id(id) + .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) + .reason(manualRedactionEntry.getReason()) + .isDictionaryEntry(manualRedactionEntry.isAddToDictionary()) + .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) + .legalBasis(manualRedactionEntry.getLegalBasis()) + .value(manualRedactionEntry.getValue()) + .type(manualRedactionEntry.getType()) + .redacted(true) + .isHint(false) + .section(null) + .sectionNumber(-1) + .manual(true) + .status(manualRedactionEntry.getStatus()) + .manualRedactionType(ManualRedactionType.ADD) + .manualRedactionUserId(manualRedactionEntry.getUser()) + .build(); + } + + + private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction, + boolean skipped) { + + if (requested) { + return dictionaryService.getRequestRemoveColor(dossierTemplateId); + } + if (skipped || !isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); + } + + + private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) { + + if (status.equals(Status.REQUESTED)) { + return dictionaryService.getRequestAddColor(dossierTemplateId); + } else if (status.equals(Status.DECLINED)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return getColor(type, dossierTemplateId); + } + + + private float[] getColor(String type, String dossierTemplateId) { + + return dictionaryService.getColor(type, dossierTemplateId); + } + + + @Data + @AllArgsConstructor + private static class ManualRedactionWrapper implements Comparable { + + private String id; + private OffsetDateTime date; + private Object item; + + + @Override + public int compareTo(ManualRedactionWrapper o) { + + return this.date.compareTo(o.date); + } + + } + +} + diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index f14eeb59..0daca578 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -5,6 +5,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncre import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; + import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -17,7 +18,6 @@ import java.util.stream.Collectors; @SuppressWarnings("PMD") public class EntitySearchUtils { - public boolean sectionContainsAny(String sectionText, Set values) { String inputString = sectionText.toLowerCase(Locale.ROOT); @@ -38,9 +38,7 @@ public class EntitySearchUtils { if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { - if (value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())) { - return true; - } + return true; } } while (startIndex > -1); } @@ -98,8 +96,8 @@ public class EntitySearchUtils { .sorted(Comparator.comparing(Entity::getStart)) .collect(Collectors.toList()); Entity firstEntity = orderedEntities.get(0); - List positionSequences = text.getSequences(firstEntity.getWord().trim(), dictionary.isCaseInsensitiveDictionary(firstEntity - .getType()), firstEntity.getTargetSequences()); + List positionSequences = text.getSequences(firstEntity.getWord() + .trim(), dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences()); for (int i = 0; i <= orderedEntities.size() - 1; i++) { try { @@ -133,6 +131,7 @@ public class EntitySearchUtils { public void addEntitiesWithHigherRank(Set entities, Set found, Dictionary dictionary) { + found.forEach(f -> addEntitiesWithHigherRank(entities, f, dictionary)); } @@ -148,9 +147,11 @@ public class EntitySearchUtils { entities.add(found); } + public void addEntitiesIgnoreRank(Set entities, Set found) { // HashSet keeps old value but we want the new. entities.removeAll(found); entities.addAll(found); } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java index b740286b..342f6b03 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java @@ -12,7 +12,7 @@ public class TextNormalizationUtilities { * @return Text without line-break hyphenation. */ public static String removeHyphenLineBreaks(String text) { - return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2"); + return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R", "$1"); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java index 73a94909..4c58bcd8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java @@ -118,7 +118,7 @@ public class ImageMergeService { //set position for merged image with values of image1 and the height of both Rectangle2D pos = new Rectangle2D.Float(); pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), rotation == 90 ? width + width2: width, rotation == 90 ? height1 : height1 + height2); - PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage()); + PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage(), image1.isHasTransparency() || image2.isHasTransparency()); // Graphics need to be disposed image1.getImage().flush(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java index 3d5b0b5b..901d8042 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java @@ -15,4 +15,8 @@ public class RedactionServiceSettings { private float maxImageCropboxRatio = 0.9f; + private int analysisVersion = 1; + + private boolean enableEntityRecognition = true; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml index 671d3b20..d8aa5ace 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml @@ -4,6 +4,7 @@ info: configuration-service.url: "http://configuration-service-v1:8080" file-management-service.url: "http://file-management-service-v1:8080" image-service.url: "http://image-service-v1:8080" +entity-recognition-service.url: "http://entity-recognition-service-v1:8080" server: port: 8080 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 609a7828..9b60a086 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -18,7 +18,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.service.StorageService; + import lombok.SneakyThrows; + import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; @@ -84,7 +86,6 @@ public class RedactionIntegrationTest { private static final String PII = "PII"; - @Autowired private RedactionController redactionController; @@ -127,6 +128,7 @@ public class RedactionIntegrationTest { private final Map rankTypeMap = new HashMap<>(); private final Colors colors = new Colors(); private final Map reanlysisVersions = new HashMap<>(); + private final Set deleted = new HashSet<>(); private final static String TEST_DOSSIER_TEMPLATE_ID = "123"; private final static String TEST_DOSSIER_ID = "123"; @@ -152,18 +154,20 @@ public class RedactionIntegrationTest { return kieServices.newKieContainer(kieModule.getReleaseId()); } + @Bean @Primary public StorageService inmemoryStorage() { + return new FileSystemBackedStorageService(); } - } @After public void cleanupStorage() { + if (this.storageService instanceof FileSystemBackedStorageService) { ((FileSystemBackedStorageService) this.storageService).clearStorage(); } @@ -180,7 +184,8 @@ public class RedactionIntegrationTest { loadDictionaryForTest(); loadTypeForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(0L); - when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse.builder() + when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse + .builder() .types(getTypeResponse()) .build()); @@ -189,7 +194,7 @@ public class RedactionIntegrationTest { .types(List.of(TypeResult.builder() .type(DOSSIER_REDACTIONS) .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .hexColor( "#ffe187") + .hexColor("#ffe187") .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) @@ -197,26 +202,42 @@ public class RedactionIntegrationTest { .build())) .build()); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); - when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(ADDRESS, false)); - when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(AUTHOR, false)); - when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SPONSOR, false)); - when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); - when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); - when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(HINT_ONLY, false)); - when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); - when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); - when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(TEST_METHOD, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(SPONSOR, false)); + when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(HINT_ONLY, false)); + when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(MUST_REDACT, false)); + when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); + when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(TEST_METHOD, false)); when(dictionaryClient.getDictionaryForType(PII, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PII, false)); - when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false)); - when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false)); - when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); - when(dictionaryClient.getDictionaryForType(PURITY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PURITY, false)); + when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); + when(dictionaryClient.getDictionaryForType(PURITY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(PURITY, false)); when(dictionaryClient.getDictionaryForType(IMAGE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(IMAGE, false)); when(dictionaryClient.getDictionaryForType(OCR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(OCR, false)); when(dictionaryClient.getDictionaryForType(LOGO, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(LOGO, false)); - when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SIGNATURE, false)); - when(dictionaryClient.getDictionaryForType(FORMULA, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FORMULA, false)); + when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(SIGNATURE, false)); + when(dictionaryClient.getDictionaryForType(FORMULA, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(FORMULA, false)); when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS, TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true)); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } @@ -478,7 +499,8 @@ public class RedactionIntegrationTest { return DictionaryResponse.builder() .hexColor(typeColorMap.get(type)) - .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type))) + .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary + .get(type))) .isHint(hintTypeMap.get(type)) .isCaseInsensitive(caseInSensitiveMap.get(type)) .isRecommendation(recommendationTypeMap.get(type)) @@ -491,7 +513,8 @@ public class RedactionIntegrationTest { List dictionaryEntries = new ArrayList<>(); entries.forEach(entry -> { - dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, false)); + dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, deleted + .contains(entry) ? true : false)); }); return dictionaryEntries; } @@ -499,6 +522,7 @@ public class RedactionIntegrationTest { @Test public void test270Rotated() { + AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf"); MemoryStats.printMemoryStats(); AnalyzeResult result = reanalyzeService.analyze(request); @@ -509,12 +533,14 @@ public class RedactionIntegrationTest { @Test @Ignore public void testLargeScannedFileOOM() { + AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf"); MemoryStats.printMemoryStats(); AnalyzeResult result = reanalyzeService.analyze(request); assertThat(result).isNotNull(); } + @Test public void testMergedImages() throws IOException { @@ -553,14 +579,13 @@ public class RedactionIntegrationTest { long rend = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (rend - rstart)); - long end = System.currentTimeMillis(); System.out.println("duration: " + (end - start)); - } + @Test @Ignore public void noExceptionShouldBeThrownForAnyFiles() throws IOException { @@ -636,7 +661,12 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); request.setExcludedPages(Set.of(1)); - request.setFileAttributes(List.of(FileAttribute.builder().id("fileAttributeId").label("Vertebrate Study").placeholder("{fileattributes.vertebrateStudy}").value("true").build())); + request.setFileAttributes(List.of(FileAttribute.builder() + .id("fileAttributeId") + .label("Vertebrate Study") + .placeholder("{fileattributes.vertebrateStudy}") + .value("true") + .build())); AnalyzeResult result = reanalyzeService.analyze(request); @@ -684,12 +714,21 @@ public class RedactionIntegrationTest { dictionary.get(AUTHOR).add("physical"); reanlysisVersions.put("physical", 2L); -// dictionary.get(VERTEBRATE).add("s-metolachlor"); -// reanlysisVersions.put("s-metolachlor", 3L); + deleted.add("David Chubb"); + deleted.add("mouse"); + + dictionary.get(FALSE_POSITIVE).add("David Chubb"); + reanlysisVersions.put("David Chubb", 3L); + + reanlysisVersions.put("mouse", 3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); start = System.currentTimeMillis(); @@ -699,16 +738,12 @@ public class RedactionIntegrationTest { .id("37eee3e9d589a5cc529bfec38c3ba479") .status(Status.APPROVED) .type("signature") - .redacted(true) - .legalBasis("Article 39(e)(1) and Article 39(e)(2) of Regulation (EC) No 178/2002") .build())); request.setManualRedactions(manualRedactions); - AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request); - redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); end = System.currentTimeMillis(); @@ -723,6 +758,20 @@ public class RedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } + + deleted.remove("mouse"); + reanlysisVersions.put("mouse", 4L); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L); + + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + reanalyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + System.out.println("hi"); } @@ -778,7 +827,6 @@ public class RedactionIntegrationTest { .status(Status.APPROVED) .build())); - manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment)); manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment)); manualRedactions.getComments().put(manualAddId, List.of(comment)); @@ -793,12 +841,10 @@ public class RedactionIntegrationTest { // manualRedactions.getEntriesToAdd().add(manualRedactionEntry); - AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); request.setManualRedactions(manualRedactions); AnalyzeResult result = reanalyzeService.analyze(request); - manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() .id("5b940b2cb401ed9f5be6fc24f6e77bcf") @@ -819,7 +865,6 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } @@ -836,7 +881,6 @@ public class RedactionIntegrationTest { System.out.println("classificationTest"); ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf"); - AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); RedactionRequest redactionRequest = RedactionRequest.builder() @@ -937,8 +981,10 @@ public class RedactionIntegrationTest { }); } + @SneakyThrows private AnalyzeRequest prepareStorage(String file) { + ClassPathResource pdfFileResource = new ClassPathResource(file); return prepareStorage(pdfFileResource.getInputStream()); @@ -970,7 +1016,6 @@ public class RedactionIntegrationTest { long start = System.currentTimeMillis(); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf"); - AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); AnalyzeResult result = reanalyzeService.analyze(request); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml index 4b511179..72e05696 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml @@ -1,6 +1,7 @@ configuration-service.url: "http://configuration-service-v1:8080" image-service.url: "http://image-service-v1:8080" file-management-service.url: "http://file-management-service-v1:8080" +entity-recognition-service.url: "http://entity-recognition-service-v1:8080" ribbon: ConnectTimeout: 600000 @@ -17,3 +18,4 @@ platform.multi-tenancy: redaction-service: enable-image-classification: false + enable-entity-recognition: false diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt index 766ebd06..3d53e6d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt @@ -1676,7 +1676,6 @@ da Silva Rejane Das R Das, R. Daughtry, CST -David Chubb David Chubb|Lorraine Britton David Clarke Davies diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt index 0bb75f29..a696cee0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt @@ -235,4 +235,5 @@ N/A No details reported Not available Test facility -TBD \ No newline at end of file +TBD +David Chubb \ No newline at end of file