From 6741b8a596fcb7a014fdbca371329f1cf46d1fd7 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 12 Jul 2021 10:33:30 +0300 Subject: [PATCH 01/46] added dictionary entries --- .../service/RedactionLogCreatorService.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 5c5e1e70..c2a32f4c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -624,9 +624,27 @@ public class RedactionLogCreatorService { processRedactionLogEntry(manualRedactions, dossierTemplateId, entry, reasonHolder); } + + handleAddToDictionary(redactionLog, manualRedactions, dossierTemplateId); + return redactionLog; } + + private void handleAddToDictionary(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) { + + for (var manualRedaction : manualRedactions.getEntriesToAdd()) { + + if (manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) { + var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); + redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + + redactionLog.getRedactionLogEntry().add(redactionLogEntry); + } + } + } + public static class PreviewReasonHolder implements ReasonHolder { private final RedactionLogEntry entry; From cc1c3122bb3a4704a8ca1ff858a35f335d7d058b Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 12 Jul 2021 10:44:42 +0300 Subject: [PATCH 02/46] fixed dictionary add corner case --- .../redaction/service/RedactionLogCreatorService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index c2a32f4c..03a5333b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -640,7 +640,15 @@ public class RedactionLogCreatorService { redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - redactionLog.getRedactionLogEntry().add(redactionLogEntry); + var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); + if (found.isPresent()) { + found.get().setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + } else { + redactionLog.getRedactionLogEntry().add(redactionLogEntry); + } + + } } } From f713ea2655bd50566e29cc4a96181c7d272b4668 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 12 Jul 2021 11:16:48 +0300 Subject: [PATCH 03/46] added some logging --- .../redaction/v1/server/controller/RedactionController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 0149d6a0..ace46234 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -160,6 +160,7 @@ public class RedactionController implements RedactionResource { @Override public RedactionLog getRedactionLogPreview(RedactionRequest redactionRequest) { + log.info("Requested preview for: {}", redactionRequest); var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); return redactionLogCreatorService.getRedactionLogPreview(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); From 79660f1623056f6cc592d48c76fba5c2f08dd304 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 12 Jul 2021 11:18:13 +0300 Subject: [PATCH 04/46] added some logging --- .../redaction/v1/server/redaction/service/DictionaryService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index 0518eb3f..71cd08b2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -163,6 +163,7 @@ public class DictionaryService { public float[] getColor(String type, String dossierTemplateId) { + log.info("requested : {} / {}",type,dossierTemplateId); DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type); if (model != null) { return model.getColor(); From 3517841d24bc69028391043157d9fcf142600ddf Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 12 Jul 2021 11:43:24 +0300 Subject: [PATCH 05/46] positions for dict entries --- .../v1/server/redaction/service/RedactionLogCreatorService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 03a5333b..a83d3751 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -639,11 +639,13 @@ public class RedactionLogCreatorService { var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + redactionLogEntry.setPositions(manualRedaction.getPositions()); var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); if (found.isPresent()) { found.get().setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + found.get().setPositions(manualRedaction.getPositions()); } else { redactionLog.getRedactionLogEntry().add(redactionLogEntry); } From 279fdc69854224d145cc4045803c9985fe932ce9 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 13 Jul 2021 11:34:08 +0300 Subject: [PATCH 06/46] added dates to model classes for manual redactions --- .../iqser/red/service/redaction/v1/model/IdRemoval.java | 8 +++++++- .../red/service/redaction/v1/model/ManualForceRedact.java | 8 +++++++- .../redaction/v1/model/ManualImageRecategorization.java | 8 +++++++- .../redaction/v1/model/ManualLegalBasisChange.java | 8 +++++++- .../service/redaction/v1/model/ManualRedactionEntry.java | 5 +++++ .../redaction/service/RedactionLogCreatorService.java | 2 ++ 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java index 2b7b506d..ff5a7329 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java @@ -5,6 +5,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; + @Data @Builder @AllArgsConstructor @@ -16,4 +18,8 @@ public class IdRemoval { private Status status; private boolean removeFromDictionary; -} \ No newline at end of file + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java index 3af345dd..c005e6d3 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java @@ -5,6 +5,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; + @Data @Builder @AllArgsConstructor @@ -16,4 +18,8 @@ public class ManualForceRedact { private Status status; private String legalBasis; -} \ No newline at end of file + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java index 7dc9120c..17e07f5c 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java @@ -5,6 +5,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; + @Data @Builder @AllArgsConstructor @@ -18,4 +20,8 @@ public class ManualImageRecategorization { private String legalBasis; private boolean redacted; -} \ No newline at end of file + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java index 4f0d211f..39210d07 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java @@ -5,6 +5,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; + @Data @Builder @AllArgsConstructor @@ -16,4 +18,8 @@ public class ManualLegalBasisChange { private Status status; private String legalBasis; -} \ No newline at end of file + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java index eb4fbecf..df14617b 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java @@ -5,6 +5,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; @@ -29,4 +30,8 @@ public class ManualRedactionEntry { private boolean addToDossierDictionary; + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index a83d3751..fe4e95c1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -641,6 +641,8 @@ public class RedactionLogCreatorService { redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); redactionLogEntry.setPositions(manualRedaction.getPositions()); + manualRedaction. + var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); if (found.isPresent()) { found.get().setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); From af46bdca90aab6aab9e3ac5c5a9808f8eea8c760 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 13 Jul 2021 11:35:18 +0300 Subject: [PATCH 07/46] handling add to dict requests --- .../service/RedactionLogCreatorService.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index fe4e95c1..aa60d857 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -635,24 +635,26 @@ public class RedactionLogCreatorService { for (var manualRedaction : manualRedactions.getEntriesToAdd()) { - if (manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) { - var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); - redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - redactionLogEntry.setPositions(manualRedaction.getPositions()); + if (manualRedaction.getProcessedDate() == null) { + + if (manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) { + var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); + redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + redactionLogEntry.setPositions(manualRedaction.getPositions()); + + + var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); + if (found.isPresent()) { + found.get().setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + found.get().setPositions(manualRedaction.getPositions()); + } else { + redactionLog.getRedactionLogEntry().add(redactionLogEntry); + } - manualRedaction. - var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); - if (found.isPresent()) { - found.get().setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - found.get().setPositions(manualRedaction.getPositions()); - } else { - redactionLog.getRedactionLogEntry().add(redactionLogEntry); } - - } } } From 2d0a6d4d29034c1afee5a122a87135614e099fbd Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 13 Jul 2021 11:39:44 +0300 Subject: [PATCH 08/46] fixed pmd --- .../service/RedactionLogCreatorService.java | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index aa60d857..ee628067 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -635,26 +635,24 @@ public class RedactionLogCreatorService { for (var manualRedaction : manualRedactions.getEntriesToAdd()) { - if (manualRedaction.getProcessedDate() == null) { - - if (manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) { - var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); - redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - redactionLogEntry.setPositions(manualRedaction.getPositions()); - - - var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); - if (found.isPresent()) { - found.get().setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - found.get().setPositions(manualRedaction.getPositions()); - } else { - redactionLog.getRedactionLogEntry().add(redactionLogEntry); - } + // not yet processed, and dictionary modifying, show in redaction-log preview + if (manualRedaction.getProcessedDate() == null && manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) { + var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); + redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + redactionLogEntry.setPositions(manualRedaction.getPositions()); + var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); + if (found.isPresent()) { + found.get().setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + found.get().setPositions(manualRedaction.getPositions()); + } else { + redactionLog.getRedactionLogEntry().add(redactionLogEntry); } + + } } } From 9da43429393a875b82e0ea73dfcd8093bede91c5 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 13 Jul 2021 12:43:07 +0300 Subject: [PATCH 09/46] changed to fonts 3.8 to fix build --- redaction-service-image-v1/src/main/docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/redaction-service-image-v1/src/main/docker/Dockerfile b/redaction-service-image-v1/src/main/docker/Dockerfile index 3ab50727..dec2d219 100644 --- a/redaction-service-image-v1/src/main/docker/Dockerfile +++ b/redaction-service-image-v1/src/main/docker/Dockerfile @@ -13,6 +13,6 @@ RUN apt-get update \ wget cabextract xfonts-utils fonts-liberation \ && rm -rf /var/lib/apt/lists/* -RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.7_all.deb -o /tmp/ttf-mscorefonts-installer_3.7_all.deb \ - && dpkg -i /tmp/ttf-mscorefonts-installer_3.7_all.deb \ - && rm /tmp/ttf-mscorefonts-installer_3.7_all.deb \ +RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb -o /tmp/ttf-mscorefonts-installer_3.8_all.deb \ + && dpkg -i /tmp/ttf-mscorefonts-installer_3.8_all.deb \ + && rm /tmp/ttf-mscorefonts-installer_3.8_all.deb \ From b42dd58824784b356b952550de92a848de412acd Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 13 Jul 2021 15:18:06 +0300 Subject: [PATCH 10/46] update dictionary before preview --- .../redaction/v1/server/controller/RedactionController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index ace46234..ebbaffc2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -161,8 +161,9 @@ public class RedactionController implements RedactionResource { public RedactionLog getRedactionLogPreview(RedactionRequest redactionRequest) { log.info("Requested preview for: {}", redactionRequest); - var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); + dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId()); + var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); return redactionLogCreatorService.getRedactionLogPreview(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); } From 58dd6841d60ad8562ae0aedf3e0aea2889fb8a4c Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 13 Jul 2021 15:35:39 +0300 Subject: [PATCH 11/46] fixed typo --- .../server/redaction/service/RedactionLogCreatorService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index ee628067..6c989a9e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -638,14 +638,14 @@ public class RedactionLogCreatorService { // not yet processed, and dictionary modifying, show in redaction-log preview if (manualRedaction.getProcessedDate() == null && manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) { var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); - redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDictionary()); redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); redactionLogEntry.setPositions(manualRedaction.getPositions()); var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); if (found.isPresent()) { - found.get().setDictionaryEntry(manualRedaction.isAddToDossierDictionary()); + found.get().setDictionaryEntry(manualRedaction.isAddToDictionary()); found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); found.get().setPositions(manualRedaction.getPositions()); } else { From 5e50c0bfde741c7780bb9048c03830cf05bbf4a8 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 13 Jul 2021 16:34:44 +0300 Subject: [PATCH 12/46] added logs for dict --- .../redaction/v1/server/redaction/service/DictionaryService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index 71cd08b2..fd460dbd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -33,6 +33,7 @@ public class DictionaryService { public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) { + log.info("Updating dictionary data for: {} / {}", dossierTemplateId, dossierId); long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER); var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId); if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) { From 71ba89306715d88ddadaa7780f4935a99bd107d6 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 14 Jul 2021 11:20:37 +0300 Subject: [PATCH 13/46] id removal modifying dict flag --- .../server/redaction/service/RedactionLogCreatorService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 6c989a9e..cd5416ae 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -146,6 +146,8 @@ public class RedactionLogCreatorService { redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setManual(true); redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); + redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); } } } @@ -298,6 +300,8 @@ public class RedactionLogCreatorService { redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setManual(true); redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); + redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); } } } @@ -326,6 +330,7 @@ public class RedactionLogCreatorService { redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setManual(true); redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + } } } From cb41187be660b5f4180c11e890bae57ebc79308a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Wed, 14 Jul 2021 12:24:27 +0200 Subject: [PATCH 14/46] RED-1814: Apply manual image recategorization before rules execution --- .../v1/model/ManualImageRecategorization.java | 2 - .../service/EntityRedactionService.java | 40 ++++++++++++++----- .../redaction/service/ReanalyzeService.java | 26 ++++++++++-- .../service/RedactionLogCreatorService.java | 7 +--- .../v1/server/RedactionIntegrationTest.java | 2 - 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java index 17e07f5c..14924878 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java @@ -17,8 +17,6 @@ public class ManualImageRecategorization { private String user; private Status status; private String type; - private String legalBasis; - private boolean redacted; private OffsetDateTime requestDate; private OffsetDateTime processedDate; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index e6d00c15..2a26bfec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,19 +1,24 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import com.iqser.red.service.redaction.v1.model.FileAttribute; +import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.model.ManualRedactions; import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.SectionArea; +import com.iqser.red.service.redaction.v1.model.Status; import com.iqser.red.service.redaction.v1.server.classification.model.*; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; + import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.kie.api.runtime.KieContainer; @@ -34,7 +39,8 @@ public class EntityRedactionService { private final SurroundingWordsService surroundingWordsService; - public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId, List fileAttributes) { + public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, + String dossierId, List fileAttributes) { dictionaryService.updateDictionary(dossierTemplateId, dossierId); KieContainer container = droolsExecutionService.updateRules(dossierTemplateId); @@ -85,7 +91,8 @@ public class EntityRedactionService { private Set findEntities(Document classifiedDoc, KieContainer kieContainer, ManualRedactions manualRedactions, Dictionary dictionary, boolean local, - Map> hintsPerSectionNumber, List fileAttributes) { + Map> hintsPerSectionNumber, + List fileAttributes) { Set documentEntities = new HashSet<>(); @@ -165,7 +172,8 @@ public class EntityRedactionService { ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, - Map> hintsPerSectionNumber, List fileAttributes) { + Map> hintsPerSectionNumber, + List fileAttributes) { List sectionSearchableTextPairs = new ArrayList<>(); @@ -334,7 +342,7 @@ public class EntityRedactionService { sectionText.setSectionNumber(sectionNumber.intValue()); sectionText.setTable(false); sectionText.setImages(images.stream() - .map(image -> convert(image, sectionNumber.intValue(), headline)) + .map(image -> convertAndRecategorize(image, sectionNumber.intValue(), headline, manualRedactions)) .collect(Collectors.toSet())); sectionText.setTextBlocks(paragraphTextBlocks); classifiedDoc.getSectionText().add(sectionText); @@ -357,7 +365,7 @@ public class EntityRedactionService { .searchableText(searchableText) .dictionary(dictionary) .images(images.stream() - .map(image -> convert(image, sectionNumber.intValue(), headline)) + .map(image -> convertAndRecategorize(image, sectionNumber.intValue(), headline, manualRedactions)) .collect(Collectors.toSet())) .fileAttributes(fileAttributes) .build(), searchableText); @@ -376,9 +384,11 @@ public class EntityRedactionService { String lowercaseInputString = searchableString.toLowerCase(); for (DictionaryModel model : dictionary.getDictionaryModels()) { if (model.isCaseInsensitive()) { - found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary())); + found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model + .isDossierDictionary())); } else { - found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary())); + found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model + .isDossierDictionary())); } } @@ -406,9 +416,9 @@ public class EntityRedactionService { } - private Image convert(PdfImage pdfImage, int sectionNumber, String headline) { + private Image convertAndRecategorize(PdfImage pdfImage, int sectionNumber, String headline, ManualRedactions manualRedactions) { - return Image.builder() + Image image = Image.builder() .type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType() .name() .toLowerCase(Locale.ROOT)) @@ -417,6 +427,18 @@ public class EntityRedactionService { .section(headline) .page(pdfImage.getPage()) .build(); + + String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); + if (manualRedactions != null && manualRedactions.getImageRecategorizations() != null) { + for (ManualImageRecategorization imageRecategorization : manualRedactions.getImageRecategorizations()) { + if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId() + .equals(imageId)) { + image.setType(imageRecategorization.getType()); + } + } + } + + return image; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index ddc67900..91f7398d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -27,6 +27,7 @@ import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.SectionArea; +import com.iqser.red.service.redaction.v1.model.Status; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; @@ -42,6 +43,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -205,6 +207,20 @@ public class ReanalyzeService { surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary); } + if (reanalysisSection.getImages() != null && !reanalysisSection.getImages() + .isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions() + .getImageRecategorizations() != null) { + for (Image image : reanalysisSection.getImages()) { + String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); + for(ManualImageRecategorization imageRecategorization: analyzeRequest.getManualRedactions() + .getImageRecategorizations()){ + if(imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId().equals(imageId)){ + image.setType(imageRecategorization.getType()); + } + } + } + } + sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() .isLocal(false) .dictionaryTypes(dictionary.getTypes()) @@ -300,9 +316,13 @@ public class ReanalyzeService { return new HashSet<>(); } - return Stream.concat(manualRedactions.getManualLegalBasisChanges().stream().map(ManualLegalBasisChange::getId), - Stream.concat(manualRedactions.getImageRecategorizations().stream().map(ManualImageRecategorization::getId), - Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)))) + return Stream.concat(manualRedactions.getManualLegalBasisChanges() + .stream() + .map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorizations() + .stream() + .map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove() + .stream() + .map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)))) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index cd5416ae..4a4a9bdf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -99,14 +99,9 @@ public class RedactionLogCreatorService { if (recategorization.getId().equals(redactionLogEntry.getId())) { String manualOverrideReason = null; if (recategorization.getStatus().equals(Status.APPROVED)) { - image.setRedaction(recategorization.isRedacted()); - redactionLogEntry.setType(recategorization.getType()); - redactionLogEntry.setHint(dictionaryService.isHint(recategorization.getType(), dossierTemplateId)); - redactionLogEntry.setRedacted(recategorization.isRedacted()); redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setLegalBasis(recategorization.getLegalBasis()); + redactionLogEntry.setType(recategorization.getType()); manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", recategorized by manual override"); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); } else if (recategorization.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to recategorize"); redactionLogEntry.setStatus(Status.REQUESTED); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 305df362..8904ea6c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -698,8 +698,6 @@ public class RedactionIntegrationTest { .id("37eee3e9d589a5cc529bfec38c3ba479") .status(Status.APPROVED) .type("signature") - .redacted(true) - .legalBasis("Article 39(e)(1) and Article 39(e)(2) of Regulation (EC) No 178/2002") .build())); request.setManualRedactions(manualRedactions); From 84db91d74dabc82306911972a0ffface1bcdff3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Wed, 14 Jul 2021 17:03:22 +0200 Subject: [PATCH 15/46] RED-1813: Fixed to large images --- .../server/parsing/PDFLinesTextStripper.java | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java index 45bcef6a..d35e51b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java @@ -189,18 +189,17 @@ public class PDFLinesTextStripper extends PDFTextStripper { COSName objectName = (COSName) arguments.get(0); PDXObject xobject = getResources().getXObject(objectName); if (xobject instanceof PDImageXObject) { - PDImageXObject pdfImage = (PDImageXObject) xobject; + PDImageXObject image = (PDImageXObject)xobject; - Rectangle2D imageBounds = calculateImagePosition(pdfImage); + Matrix ctmNew = getGraphicsState().getCurrentTransformationMatrix(); - Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds - .getWidth(), (float) imageBounds.getHeight()); + Rectangle2D rect = new Rectangle2D.Float(ctmNew.getTranslateX(), ctmNew.getTranslateY(), ctmNew.getScaleX(), ctmNew.getScaleY()); // Memory Hack - sofReference kills me - FieldUtils.writeField(pdfImage, "cachedImageSubsampling", -1, true); + FieldUtils.writeField(image, "cachedImageSubsampling", -1, true); if (rect.getHeight() > 2 && rect.getWidth() > 2) { - this.images.add(new PdfImage(pdfImage.getImage(), rect, pageNumber)); + this.images.add(new PdfImage(image.getImage(), rect, pageNumber)); } } } catch (Exception e) { @@ -209,21 +208,6 @@ public class PDFLinesTextStripper extends PDFTextStripper { } - private Rectangle2D calculateImagePosition(PDImageXObject pdfImage) throws IOException { - - Matrix ctm = getGraphicsState().getCurrentTransformationMatrix(); - - Rectangle2D imageBounds = pdfImage.getImage().getRaster().getBounds(); - - AffineTransform imageTransform = new AffineTransform(ctm.createAffineTransform()); - imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight()); - imageTransform.translate(0, -pdfImage.getHeight()); - - AffineTransform pageTransform = new AffineTransform(); - pageTransform.concatenate(imageTransform); - - return pageTransform.createTransformedShape(imageBounds).getBounds2D(); - } private float floatValue(COSBase value) { From 93844d8e76e0806cf00292b591a9f97ee8e61230 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 19 Jul 2021 11:39:19 +0300 Subject: [PATCH 16/46] fixed undo after reanalysis --- .../service/RedactionLogCreatorService.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 4a4a9bdf..7bd49fbc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -613,6 +613,7 @@ public class RedactionLogCreatorService { } + Set entriesToRemoveBecauseOfUndoActions = new HashSet<>(); for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { var reasonHolder = new PreviewReasonHolder(entry); @@ -622,14 +623,66 @@ public class RedactionLogCreatorService { } processRedactionLogEntry(manualRedactions, dossierTemplateId, entry, reasonHolder); + var shouldRemove = postProcessRedactionLogEntryForDeletedManualRedactions(manualRedactions, entry); + if (shouldRemove) { + entriesToRemoveBecauseOfUndoActions.add(entry.getId()); + } } + redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry() + .stream().filter(entry -> !entriesToRemoveBecauseOfUndoActions.contains(entry.getId())).collect(Collectors.toList())); + handleAddToDictionary(redactionLog, manualRedactions, dossierTemplateId); return redactionLog; } + private boolean postProcessRedactionLogEntryForDeletedManualRedactions(ManualRedactions manualRedactions, RedactionLogEntry entry) { + if (entry.isManual()) { + + if (entry.getManualRedactionType() == ManualRedactionType.ADD) { + var manualRedactionExists = manualRedactions.getEntriesToAdd().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); + return !manualRedactionExists; + } + + if (entry.getManualRedactionType() == ManualRedactionType.FORCE_REDACT) { + var forceRedactExists = manualRedactions.getForceRedacts().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); + if (!forceRedactExists) { + entry.setRedacted(false); + entry.setManual(false); + entry.setStatus(null); + } + } + + if (entry.getManualRedactionType() == ManualRedactionType.REMOVE) { + var removeIdExists = manualRedactions.getIdsToRemove().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); + if (!removeIdExists) { + entry.setRedacted(true); + entry.setManual(false); + entry.setStatus(null); + } + } + + // Cannot undo Already Approved change because UI won't allow it for now + if (Status.REQUESTED.equals(entry.getStatus())) { + if (entry.getManualRedactionType() == ManualRedactionType.RECATEGORIZE) { + entry.setManual(false); + entry.setStatus(null); + entry.setRecategorizationType(null); + } + if (entry.getManualRedactionType() == ManualRedactionType.LEGAL_BASIS_CHANGE) { + entry.setManual(false); + entry.setStatus(null); + entry.setLegalBasisChangeValue(null); + } + + } + } + + return false; + } + private void handleAddToDictionary(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) { From 0b89c55a1d0aacf9fd84efdb5c59a881d4ae1a4d Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 19 Jul 2021 11:57:29 +0300 Subject: [PATCH 17/46] fixed undo after reanalysis --- .../service/RedactionLogCreatorService.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 7bd49fbc..273a4f69 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -666,17 +666,22 @@ public class RedactionLogCreatorService { // Cannot undo Already Approved change because UI won't allow it for now if (Status.REQUESTED.equals(entry.getStatus())) { - if (entry.getManualRedactionType() == ManualRedactionType.RECATEGORIZE) { - entry.setManual(false); - entry.setStatus(null); - entry.setRecategorizationType(null); + var recategorizeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); + if (!recategorizeExists) { + if (entry.getManualRedactionType() == ManualRedactionType.RECATEGORIZE) { + entry.setManual(false); + entry.setStatus(null); + entry.setRecategorizationType(null); + } } - if (entry.getManualRedactionType() == ManualRedactionType.LEGAL_BASIS_CHANGE) { - entry.setManual(false); - entry.setStatus(null); - entry.setLegalBasisChangeValue(null); + var legalBasisChangeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); + if (!legalBasisChangeExists) { + if (entry.getManualRedactionType() == ManualRedactionType.LEGAL_BASIS_CHANGE) { + entry.setManual(false); + entry.setStatus(null); + entry.setLegalBasisChangeValue(null); + } } - } } From d4486ea6f4d31d68a2c75ec5e192f2a3bd62443c Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 19 Jul 2021 11:59:53 +0300 Subject: [PATCH 18/46] fixed PMD --- .../service/RedactionLogCreatorService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 273a4f69..8518891f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -666,17 +666,17 @@ public class RedactionLogCreatorService { // Cannot undo Already Approved change because UI won't allow it for now if (Status.REQUESTED.equals(entry.getStatus())) { - var recategorizeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!recategorizeExists) { - if (entry.getManualRedactionType() == ManualRedactionType.RECATEGORIZE) { + if (entry.getManualRedactionType() == ManualRedactionType.RECATEGORIZE) { + var recategorizeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); + if (!recategorizeExists) { entry.setManual(false); entry.setStatus(null); entry.setRecategorizationType(null); } } - var legalBasisChangeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!legalBasisChangeExists) { - if (entry.getManualRedactionType() == ManualRedactionType.LEGAL_BASIS_CHANGE) { + if (entry.getManualRedactionType() == ManualRedactionType.LEGAL_BASIS_CHANGE) { + var legalBasisChangeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); + if (!legalBasisChangeExists) { entry.setManual(false); entry.setStatus(null); entry.setLegalBasisChangeValue(null); From a39bbd9afa814577543a33c35dd27c96ec993e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Mon, 19 Jul 2021 13:12:20 +0200 Subject: [PATCH 19/46] RED-1184: Always find reanaylsis sections with caseinsentive values to remove values if type has changed from caseinsensitive to casesensitive --- .../server/redaction/utils/EntitySearchUtils.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index f14eeb59..0daca578 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -5,6 +5,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncre import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; + import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -17,7 +18,6 @@ import java.util.stream.Collectors; @SuppressWarnings("PMD") public class EntitySearchUtils { - public boolean sectionContainsAny(String sectionText, Set values) { String inputString = sectionText.toLowerCase(Locale.ROOT); @@ -38,9 +38,7 @@ public class EntitySearchUtils { if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { - if (value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())) { - return true; - } + return true; } } while (startIndex > -1); } @@ -98,8 +96,8 @@ public class EntitySearchUtils { .sorted(Comparator.comparing(Entity::getStart)) .collect(Collectors.toList()); Entity firstEntity = orderedEntities.get(0); - List positionSequences = text.getSequences(firstEntity.getWord().trim(), dictionary.isCaseInsensitiveDictionary(firstEntity - .getType()), firstEntity.getTargetSequences()); + List positionSequences = text.getSequences(firstEntity.getWord() + .trim(), dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences()); for (int i = 0; i <= orderedEntities.size() - 1; i++) { try { @@ -133,6 +131,7 @@ public class EntitySearchUtils { public void addEntitiesWithHigherRank(Set entities, Set found, Dictionary dictionary) { + found.forEach(f -> addEntitiesWithHigherRank(entities, f, dictionary)); } @@ -148,9 +147,11 @@ public class EntitySearchUtils { entities.add(found); } + public void addEntitiesIgnoreRank(Set entities, Set found) { // HashSet keeps old value but we want the new. entities.removeAll(found); entities.addAll(found); } + } From e809004f1638f4982856e6c7fded719894adb3e8 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 19 Jul 2021 14:57:59 +0300 Subject: [PATCH 20/46] add comments --- .../server/redaction/service/RedactionLogCreatorService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 8518891f..b1707ae6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -627,6 +627,9 @@ public class RedactionLogCreatorService { if (shouldRemove) { entriesToRemoveBecauseOfUndoActions.add(entry.getId()); } + + + entry.setComments(manualRedactions.getComments().get(entry.getId()); } redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry() From e9c5b955805b42e3e62ae0aad7affcad05610833 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 19 Jul 2021 16:30:29 +0300 Subject: [PATCH 21/46] fixed compile err --- .../v1/server/redaction/service/RedactionLogCreatorService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index b1707ae6..18e23420 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -629,7 +629,7 @@ public class RedactionLogCreatorService { } - entry.setComments(manualRedactions.getComments().get(entry.getId()); + entry.setComments(manualRedactions.getComments().get(entry.getId())); } redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry() From ba0cde241fd11ed6fe0bd623561470a108e1b89f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Tue, 20 Jul 2021 10:01:49 +0200 Subject: [PATCH 22/46] RED-1813: Added imageHasTransparency to redaction log --- .../red/service/redaction/v1/model/RedactionLogEntry.java | 1 + .../redaction/v1/server/parsing/PDFLinesTextStripper.java | 3 +-- .../service/redaction/v1/server/redaction/model/Image.java | 1 + .../service/redaction/v1/server/redaction/model/PdfImage.java | 4 +++- .../v1/server/redaction/service/EntityRedactionService.java | 1 + .../v1/server/redaction/service/ReanalyzeService.java | 1 + .../server/redaction/service/RedactionLogCreatorService.java | 1 + .../redaction/v1/server/segmentation/ImageMergeService.java | 2 +- 8 files changed, 10 insertions(+), 4 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index e347f0b3..e837ddf9 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -46,6 +46,7 @@ public class RedactionLogEntry { private int endOffset; private boolean isImage; + private boolean imageHasTransparency; private boolean isDossierDictionaryEntry; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java index d35e51b9..d3f6f2a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java @@ -190,7 +190,6 @@ public class PDFLinesTextStripper extends PDFTextStripper { PDXObject xobject = getResources().getXObject(objectName); if (xobject instanceof PDImageXObject) { PDImageXObject image = (PDImageXObject)xobject; - Matrix ctmNew = getGraphicsState().getCurrentTransformationMatrix(); Rectangle2D rect = new Rectangle2D.Float(ctmNew.getTranslateX(), ctmNew.getTranslateY(), ctmNew.getScaleX(), ctmNew.getScaleY()); @@ -199,7 +198,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { FieldUtils.writeField(image, "cachedImageSubsampling", -1, true); if (rect.getHeight() > 2 && rect.getWidth() > 2) { - this.images.add(new PdfImage(image.getImage(), rect, pageNumber)); + this.images.add(new PdfImage(image.getImage(), rect, pageNumber, image.getImage().getColorModel().hasAlpha())); } } } catch (Exception e) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java index 5aab9c7a..63d382d5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java @@ -20,5 +20,6 @@ public class Image implements ReasonHolder { private int sectionNumber; private String section; private int page; + private boolean hasTransparency; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java index 1631717f..dbb390c0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java @@ -18,14 +18,16 @@ public class PdfImage { private RedRectangle2D position; private ImageType imageType; private boolean isAppendedToParagraph; + private boolean hasTransparency; @NonNull private int page; - public PdfImage(BufferedImage image, Rectangle2D position, int page) { + public PdfImage(BufferedImage image, Rectangle2D position, int page, boolean hasTransparency) { this.image = image; this.position = new RedRectangle2D(position.getX(), position.getY(), position.getWidth(), position.getHeight()); this.page = page; + this.hasTransparency = hasTransparency; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 2a26bfec..389b7186 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -426,6 +426,7 @@ public class EntityRedactionService { .sectionNumber(sectionNumber) .section(headline) .page(pdfImage.getPage()) + .hasTransparency(pdfImage.isHasTransparency()) .build(); String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 91f7398d..fa3f388c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -338,6 +338,7 @@ public class ReanalyzeService { .sectionNumber(entry.getSectionNumber()) .section(entry.getSection()) .page(position.getPage()) + .hasTransparency(entry.isImageHasTransparency()) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 18e23420..885326c9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -83,6 +83,7 @@ public class RedactionLogCreatorService { .getWidth(), (float) image.getPosition().getHeight(), pageNumber))) .sectionNumber(image.getSectionNumber()) .section(image.getSection()) + .imageHasTransparency(image.isHasTransparency()) .build(); processImageEntry(manualRedactions, dossierTemplateId, image, redactionLogEntry); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java index 73a94909..4c58bcd8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java @@ -118,7 +118,7 @@ public class ImageMergeService { //set position for merged image with values of image1 and the height of both Rectangle2D pos = new Rectangle2D.Float(); pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), rotation == 90 ? width + width2: width, rotation == 90 ? height1 : height1 + height2); - PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage()); + PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage(), image1.isHasTransparency() || image2.isHasTransparency()); // Graphics need to be disposed image1.getImage().flush(); From 1ac294fc0197990936ba62b3889474f63844bc21 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 21 Jul 2021 15:58:52 +0300 Subject: [PATCH 23/46] redaction-log preview generation and removal of manual redactions from redaction-log --- .../redaction/v1/model/AnalyzeRequest.java | 2 - .../v1/model/ManualRedactionEntry.java | 5 +- .../redaction/v1/model/RedactionLog.java | 7 + .../redaction/v1/model/RedactionLogEntry.java | 2 + .../redaction/v1/model/SectionRectangle.java | 1 + .../v1/resources/RedactionResource.java | 2 +- .../controller/RedactionController.java | 8 +- .../redaction/service/DictionaryService.java | 2 +- .../service/EntityRedactionService.java | 40 +- .../redaction/service/ReanalyzeService.java | 102 +--- .../service/RedactionLogCreatorService.java | 547 +----------------- .../service/RedactionLogMergeService.java | 241 ++++++++ 12 files changed, 312 insertions(+), 647 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 7a891277..4a7be09b 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -7,9 +7,7 @@ import lombok.NoArgsConstructor; import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; @Data diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java index df14617b..69eb03cf 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java @@ -23,11 +23,8 @@ public class ManualRedactionEntry { private String legalBasis; private List positions = new ArrayList<>(); private Status status; + private boolean addToDictionary; - - private String section; - private int sectionNumber; - private boolean addToDossierDictionary; private OffsetDateTime requestDate; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index 71a8413e..276363f8 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -10,6 +10,13 @@ import java.util.List; @AllArgsConstructor public class RedactionLog { + + /** + * Version 0 Redaction Logs have manual redactions merged inside them + * Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime. + */ + private long computationalVersion; + private List redactionLogEntry; private List legalBasis; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index e837ddf9..9f7a0146 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -9,6 +9,8 @@ import lombok.NoArgsConstructor; import java.util.ArrayList; import java.util.List; + + @Data @Builder @NoArgsConstructor diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java index 38031b36..3a21bb47 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java @@ -30,4 +30,5 @@ public class SectionRectangle { private int numberOfParts; private List tableCells; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index 0a6037a6..3abe15f5 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -31,6 +31,6 @@ public interface RedactionResource { void testRules(@RequestBody String rules); @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE) - RedactionLog getRedactionLogPreview(@RequestBody RedactionRequest redactionRequest); + RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index ebbaffc2..047538cf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -13,7 +13,7 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; -import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; +import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; @@ -41,7 +41,7 @@ public class RedactionController implements RedactionResource { private final AnnotationService annotationService; private final PdfSegmentationService pdfSegmentationService; private final RedactionStorageService redactionStorageService; - private final RedactionLogCreatorService redactionLogCreatorService; + private final RedactionLogMergeService redactionLogMergeService; public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { @@ -158,13 +158,13 @@ public class RedactionController implements RedactionResource { } @Override - public RedactionLog getRedactionLogPreview(RedactionRequest redactionRequest) { + public RedactionLog getRedactionLog(RedactionRequest redactionRequest) { log.info("Requested preview for: {}", redactionRequest); dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId()); var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); - return redactionLogCreatorService.getRedactionLogPreview(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); + return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index fd460dbd..5afff568 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -218,7 +218,7 @@ public class DictionaryService { public float[] getRequestRemoveColor(String dossierTemplateId) { - return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor(); + return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestRemoveColor(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 389b7186..ea1079a4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,13 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.redaction.v1.model.FileAttribute; -import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; -import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; -import com.iqser.red.service.redaction.v1.model.ManualRedactions; -import com.iqser.red.service.redaction.v1.model.Point; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.SectionArea; -import com.iqser.red.service.redaction.v1.model.Status; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.*; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.*; @@ -15,10 +8,8 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; - import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.kie.api.runtime.KieContainer; @@ -103,9 +94,9 @@ public class EntityRedactionService { List tables = paragraph.getTables(); for (Table table : tables) { if (table.getColCount() == 2) { - sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); + sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); } else { - sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); + sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); } sectionNumber.incrementAndGet(); } @@ -169,7 +160,6 @@ public class EntityRedactionService { private List processTablePerRow(Document classifiedDoc, Table table, - ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber, @@ -198,7 +188,6 @@ public class EntityRedactionService { sectionText.getSectionAreas().add(sectionArea); sectionText.getTextBlocks().addAll(cell.getTextBlocks()); - addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue()); int cellStart = start; if (!cell.isHeaderCell()) { @@ -259,7 +248,6 @@ public class EntityRedactionService { private List processTableAsOneText(Document classifiedDoc, Table table, - ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber, @@ -288,7 +276,6 @@ public class EntityRedactionService { for (TextBlock textBlock : cell.getTextBlocks()) { entireTableText.addAll(textBlock.getSequences()); } - addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue()); } } @@ -348,7 +335,6 @@ public class EntityRedactionService { classifiedDoc.getSectionText().add(sectionText); } - addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue()); Set entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local); surroundingWordsService.addSurroundingText(entities, searchableText, dictionary); @@ -396,26 +382,6 @@ public class EntityRedactionService { } - private void addSectionToManualRedactions(List textBlocks, ManualRedactions manualRedactions, - String section, int sectionNumber) { - - if (manualRedactions == null || manualRedactions.getEntriesToAdd().isEmpty()) { - return; - } - - for (TextBlock textBlock : textBlocks) { - for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) { - for (Rectangle rectangle : manualRedactionEntry.getPositions()) { - if (textBlock.contains(rectangle)) { - manualRedactionEntry.setSection(section); - manualRedactionEntry.setSectionNumber(sectionNumber); - } - } - } - } - } - - private Image convertAndRecategorize(PdfImage pdfImage, int sectionNumber, String headline, ManualRedactions manualRedactions) { Image image = Image.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index fa3f388c..17dfdf4b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -1,55 +1,28 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.RequestBody; - import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; -import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import com.iqser.red.service.redaction.v1.model.Comment; -import com.iqser.red.service.redaction.v1.model.IdRemoval; -import com.iqser.red.service.redaction.v1.model.ManualForceRedact; -import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; -import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange; -import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; -import com.iqser.red.service.redaction.v1.model.ManualRedactions; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.model.SectionArea; -import com.iqser.red.service.redaction.v1.model.Status; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; -import com.iqser.red.service.redaction.v1.server.redaction.model.Section; -import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; +import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; - import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.kie.api.runtime.KieContainer; +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.RequestBody; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Service @@ -87,13 +60,13 @@ public class ReanalyzeService { entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest .getDossierId(), analyzeRequest.getFileAttributes()); - redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest + redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest .getDossierTemplateId()); log.info("Redaction analysis successful..."); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() + var redactionLog = new RedactionLog(1, classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() .getDossierTemplateVersion(), classifiedDoc.getDictionaryVersion() .getDossierVersion(), classifiedDoc.getRulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); @@ -132,20 +105,12 @@ public class ReanalyzeService { DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions()); - Map> comments = null; - Set manualAdds = null; - - if (analyzeRequest.getManualRedactions() != null) { - // TODO comments will be removed from redactionLog, so we ignore this first. - comments = analyzeRequest.getManualRedactions().getComments(); - manualAdds = analyzeRequest.getManualRedactions().getEntriesToAdd(); - } + Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set sectionsToReanalyse = new HashSet<>(); Map> imageEntries = new HashMap<>(); for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) { + if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { sectionsToReanalyse.add(entry.getSectionNumber()); } if (entry.isImage() || entry.getType().equals("image")) { @@ -159,23 +124,11 @@ public class ReanalyzeService { sectionsToReanalyse.add(sectionText.getSectionNumber()); } - if (manualAdds != null) { - for (SectionArea sectionArea : sectionText.getSectionAreas()) { - for (ManualRedactionEntry manualAdd : manualAdds) { - for (Rectangle manualPosition : manualAdd.getPositions()) { - if (sectionArea.contains(manualPosition)) { - manualAdd.setSection(sectionText.getHeadline()); - manualAdd.setSectionNumber(sectionText.getSectionNumber()); - } - } - } - } - } } log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest); - if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) { + if (sectionsToReanalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); } @@ -212,9 +165,9 @@ public class ReanalyzeService { .getImageRecategorizations() != null) { for (Image image : reanalysisSection.getImages()) { String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); - for(ManualImageRecategorization imageRecategorization: analyzeRequest.getManualRedactions() - .getImageRecategorizations()){ - if(imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId().equals(imageId)){ + for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() + .getImageRecategorizations()) { + if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId().equals(imageId)) { image.setType(imageRecategorization.getType()); } } @@ -270,17 +223,15 @@ public class ReanalyzeService { List newRedactionLogEntries = new ArrayList<>(); for (int page = 1; page <= text.getNumberOfPages(); page++) { if (entitiesPerPage.get(page) != null) { - newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest + newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, page, analyzeRequest .getDossierTemplateId())); } if (imagesPerPage.get(page) != null) { - newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest + newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, page, analyzeRequest .getDossierTemplateId())); } - newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest - .getDossierTemplateId())); } redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); @@ -310,7 +261,7 @@ public class ReanalyzeService { } - private Set getForceAndRemoveIds(ManualRedactions manualRedactions) { + private Set getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) { if (manualRedactions == null) { return new HashSet<>(); @@ -345,15 +296,10 @@ public class ReanalyzeService { private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { - redactionLog.getRedactionLogEntry().forEach(entry -> { - entry.getPositions().forEach(pos -> { - if (excludedPages != null && excludedPages.contains(pos.getPage())) { - entry.setExcluded(true); - } else { - entry.setExcluded(false); - } - }); - }); + redactionLog.getRedactionLogEntry().forEach(entry -> + entry.getPositions().forEach(pos -> + entry.setExcluded(excludedPages != null && excludedPages.contains(pos.getPage())) + )); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 885326c9..ec1f78db 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -1,6 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.model.CellRectangle; +import com.iqser.red.service.redaction.v1.model.Point; +import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.SectionRectangle; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; @@ -9,17 +13,19 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.ReasonHolder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import lombok.RequiredArgsConstructor; import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; @Service @@ -29,10 +35,7 @@ public class RedactionLogCreatorService { private final DictionaryService dictionaryService; - public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions, - String dossierTemplateId) { - - Set manualRedactionPages = getManualRedactionPages(manualRedactions); + public void createRedactionLog(Document classifiedDoc, int numberOfPages, String dossierTemplateId) { for (int page = 1; page <= numberOfPages; page++) { @@ -40,24 +43,18 @@ public class RedactionLogCreatorService { if (classifiedDoc.getEntities().get(page) != null) { classifiedDoc.getRedactionLogEntities() - .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId)); - } - - if (manualRedactionPages.contains(page)) { - classifiedDoc.getRedactionLogEntities() - .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId)); + .addAll(addEntries(classifiedDoc.getEntities(), page, dossierTemplateId)); } if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) { classifiedDoc.getRedactionLogEntities() - .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId)); + .addAll(addImageEntries(classifiedDoc.getImages(), page, dossierTemplateId)); } } } - public List addImageEntries(Map> images, ManualRedactions manualRedactions, - int pageNumber, String dossierTemplateId) { + public List addImageEntries(Map> images, int pageNumber, String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -67,7 +64,7 @@ public class RedactionLogCreatorService { RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() .id(id) - .color(getColorForImage(image.getType(), dossierTemplateId, false, image.isRedaction())) + .color(getColor(image.getType(), dossierTemplateId, image.isRedaction())) .isImage(true) .type(image.getType()) .redacted(image.isRedaction()) @@ -86,7 +83,6 @@ public class RedactionLogCreatorService { .imageHasTransparency(image.isHasTransparency()) .build(); - processImageEntry(manualRedactions, dossierTemplateId, image, redactionLogEntry); redactionLogEntities.add(redactionLogEntry); } @@ -94,135 +90,8 @@ public class RedactionLogCreatorService { return redactionLogEntities; } - private void processImageEntry(ManualRedactions manualRedactions, String dossierTemplateId, ReasonHolder image, RedactionLogEntry redactionLogEntry) { - if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { - for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { - if (recategorization.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (recategorization.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setType(recategorization.getType()); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", recategorized by manual override"); - } else if (recategorization.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to recategorize"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setRecategorizationType(recategorization.getType()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); - } - } - } - - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - image.setRedaction(false); - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to remove"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - } - } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - image.setRedaction(true); - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", forced by manual override"); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to force redact"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - } - } - } - - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis was manually changed"); - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis change requested"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } - } - } - } - - - private Set getManualRedactionPages(ManualRedactions manualRedactions) { - - Set manualRedactionPages = new HashSet<>(); - - if (manualRedactions == null) { - return manualRedactionPages; - } - - manualRedactions.getEntriesToAdd().forEach(entry -> { - entry.getPositions().forEach(pos -> { - manualRedactionPages.add(pos.getPage()); - }); - }); - return manualRedactionPages; - } - - - public List addEntries(Map> entities, ManualRedactions manualRedactions, - int page, String dossierTemplateId) { + public List addEntries(Map> entities, int page, String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -245,7 +114,6 @@ public class RedactionLogCreatorService { } redactionLogEntry.setId(entityPositionSequence.getId()); - processRedactionLogEntry(manualRedactions, dossierTemplateId, redactionLogEntry, entity); if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) { List rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences() @@ -269,112 +137,6 @@ public class RedactionLogCreatorService { return redactionLogEntities; } - private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry, ReasonHolder reasonHolder) { - - List comments = null; - - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(redactionLogEntry.getId())) { - comments = manualRedactions.getComments().get(manualRemoval.getId()); - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - reasonHolder.setRedaction(false); - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to remove"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - } - } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - reasonHolder.setRedaction(true); - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", forced by manual override"); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to force redact"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - - } - } - } - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis was manually changed"); - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis change requested"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } - } - } - - - if (manualRedactions != null) { - comments = manualRedactions.getComments().get(redactionLogEntry.getId()); - } - - redactionLogEntry.setComments(comments); - } - - private String mergeReasonIfNecessary(String currentReason, String addition) { - if (currentReason != null) { - if (!currentReason.contains(addition)) { - return currentReason + addition; - } - return currentReason; - } else { - return ""; - } - } - private List getRectanglesPerLine(List textPositions, int page) { @@ -403,73 +165,10 @@ public class RedactionLogCreatorService { } - public List addManualAddEntries(Set manualAdds, - Map> comments, int page, - String dossierTemplateId) { - - List redactionLogEntities = new ArrayList<>(); - - if (manualAdds == null) { - return redactionLogEntities; - } - - for (ManualRedactionEntry manualRedactionEntry : manualAdds) { - - String id = manualRedactionEntry.getId(); - - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId); - - List rectanglesOnPage = new ArrayList<>(); - for (Rectangle rectangle : manualRedactionEntry.getPositions()) { - if (page == rectangle.getPage()) { - rectanglesOnPage.add(rectangle); - redactionLogEntry.getPositions().add(rectangle); - } - } - - redactionLogEntry.setComments(comments.get(id)); - if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) { - redactionLogEntities.add(redactionLogEntry); - } - } - - return redactionLogEntities; - } - - - private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { - - return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary(); - } - - - private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, - String dossierTemplateId) { - - return RedactionLogEntry.builder() - .id(id) - .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) - .reason(manualRedactionEntry.getReason()) - .legalBasis(manualRedactionEntry.getLegalBasis()) - .value(manualRedactionEntry.getValue()) - .type(manualRedactionEntry.getType()) - .redacted(true) - .isHint(false) - .section(manualRedactionEntry.getSection()) - .sectionNumber(manualRedactionEntry.getSectionNumber()) - .manual(true) - .status(manualRedactionEntry.getStatus()) - .manualRedactionType(ManualRedactionType.ADD) - .isDictionaryEntry(false) - .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) - .build(); - } - - private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) { return RedactionLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, false, entity.isRedaction())) + .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction())) .reason(entity.getRedactionReason()) .legalBasis(entity.getLegalBasis()) .value(entity.getWord()) @@ -490,59 +189,6 @@ public class RedactionLogCreatorService { } - private float[] getColor(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) { - - if (requestedToRemove) { - return dictionaryService.getRequestRemoveColor(dossierTemplateId); - } - if (!isRedaction && !isHint(type, dossierTemplateId)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) { - - if (status.equals(Status.REQUESTED)) { - return dictionaryService.getRequestAddColor(dossierTemplateId); - } else if (status.equals(Status.DECLINED)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return getColor(type, dossierTemplateId); - } - - - private float[] getColor(String type, String dossierTemplateId) { - - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private float[] getColorForImage(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) { - - if (requestedToRemove) { - return dictionaryService.getRequestRemoveColor(dossierTemplateId); - } - if (!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private boolean isHint(String type, String dossierTemplateId) { - - return dictionaryService.isHint(type, dossierTemplateId); - } - - - private boolean isRecommendation(String type, String dossierTemplateId) { - - return dictionaryService.isRecommendation(type, dossierTemplateId); - } - - private void addSectionGrid(Document classifiedDoc, int page) { for (Paragraph paragraph : classifiedDoc.getParagraphs()) { @@ -586,165 +232,26 @@ public class RedactionLogCreatorService { } } - public RedactionLog getRedactionLogPreview(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { + private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { - var manualRedactionPages = getManualRedactionPages(manualRedactions); - - // generate all manual entries - var manualRedactionLogEntries = new HashMap(); - for (var page : manualRedactionPages) { - - var pageEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId); - - for (var entry : pageEntries) { - manualRedactionLogEntries.put(entry.getId(), entry); - } + if (!isRedaction && !isHint(type, dossierTemplateId)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); } - - for (var manualEntry : manualRedactionLogEntries.values()) { - var existingEntry = redactionLog.getRedactionLogEntry().stream().filter(e -> e.getId().equals(manualEntry.getId())).findAny(); - if (existingEntry.isPresent()) { - // if it has already been processed of sorts, update it - BeanUtils.copyProperties(manualEntry, existingEntry.get()); - } else { - // not yet in the redaction-log - add it - redactionLog.getRedactionLogEntry().add(manualEntry); - } - } - - - Set entriesToRemoveBecauseOfUndoActions = new HashSet<>(); - for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - - var reasonHolder = new PreviewReasonHolder(entry); - - if (entry.isImage()) { - processImageEntry(manualRedactions, dossierTemplateId, reasonHolder, entry); - } - - processRedactionLogEntry(manualRedactions, dossierTemplateId, entry, reasonHolder); - var shouldRemove = postProcessRedactionLogEntryForDeletedManualRedactions(manualRedactions, entry); - if (shouldRemove) { - entriesToRemoveBecauseOfUndoActions.add(entry.getId()); - } - - - entry.setComments(manualRedactions.getComments().get(entry.getId())); - } - - redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry() - .stream().filter(entry -> !entriesToRemoveBecauseOfUndoActions.contains(entry.getId())).collect(Collectors.toList())); - - - handleAddToDictionary(redactionLog, manualRedactions, dossierTemplateId); - - return redactionLog; - } - - private boolean postProcessRedactionLogEntryForDeletedManualRedactions(ManualRedactions manualRedactions, RedactionLogEntry entry) { - if (entry.isManual()) { - - if (entry.getManualRedactionType() == ManualRedactionType.ADD) { - var manualRedactionExists = manualRedactions.getEntriesToAdd().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - return !manualRedactionExists; - } - - if (entry.getManualRedactionType() == ManualRedactionType.FORCE_REDACT) { - var forceRedactExists = manualRedactions.getForceRedacts().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!forceRedactExists) { - entry.setRedacted(false); - entry.setManual(false); - entry.setStatus(null); - } - } - - if (entry.getManualRedactionType() == ManualRedactionType.REMOVE) { - var removeIdExists = manualRedactions.getIdsToRemove().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!removeIdExists) { - entry.setRedacted(true); - entry.setManual(false); - entry.setStatus(null); - } - } - - // Cannot undo Already Approved change because UI won't allow it for now - if (Status.REQUESTED.equals(entry.getStatus())) { - if (entry.getManualRedactionType() == ManualRedactionType.RECATEGORIZE) { - var recategorizeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!recategorizeExists) { - entry.setManual(false); - entry.setStatus(null); - entry.setRecategorizationType(null); - } - } - if (entry.getManualRedactionType() == ManualRedactionType.LEGAL_BASIS_CHANGE) { - var legalBasisChangeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!legalBasisChangeExists) { - entry.setManual(false); - entry.setStatus(null); - entry.setLegalBasisChangeValue(null); - } - } - } - } - - return false; + return dictionaryService.getColor(type, dossierTemplateId); } - private void handleAddToDictionary(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) { + private boolean isHint(String type, String dossierTemplateId) { - for (var manualRedaction : manualRedactions.getEntriesToAdd()) { - - // not yet processed, and dictionary modifying, show in redaction-log preview - if (manualRedaction.getProcessedDate() == null && manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) { - var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); - redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - redactionLogEntry.setPositions(manualRedaction.getPositions()); - - - var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); - if (found.isPresent()) { - found.get().setDictionaryEntry(manualRedaction.isAddToDictionary()); - found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - found.get().setPositions(manualRedaction.getPositions()); - } else { - redactionLog.getRedactionLogEntry().add(redactionLogEntry); - } - - - } - } + return dictionaryService.isHint(type, dossierTemplateId); } - public static class PreviewReasonHolder implements ReasonHolder { - private final RedactionLogEntry entry; + private boolean isRecommendation(String type, String dossierTemplateId) { - public PreviewReasonHolder(RedactionLogEntry entry) { - this.entry = entry; - } - - @Override - public String getRedactionReason() { - return entry.getReason(); - } - - @Override - public void setRedactionReason(String reason) { - entry.setReason(reason); - } - - @Override - public boolean isRedaction() { - return entry.isRedacted(); - } - - @Override - public void setRedaction(boolean value) { - entry.setRedacted(value); - } + return dictionaryService.isRecommendation(type, dossierTemplateId); } + + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java new file mode 100644 index 00000000..ac19c236 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -0,0 +1,241 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import com.iqser.red.service.redaction.v1.model.*; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service +@RequiredArgsConstructor +public class RedactionLogMergeService { + + private final DictionaryService dictionaryService; + + + public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { + + if (manualRedactions != null) { + + var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); + + redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); + + for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { + processRedactionLogEntry(manualRedactions, dossierTemplateId, entry); + entry.setComments(manualRedactions.getComments().get(entry.getId())); + } + + } + + return redactionLog; + } + + + private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry) { + + List comments = null; + + + if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { + for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { + if (recategorization.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (recategorization.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setType(recategorization.getType()); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); + } else if (recategorization.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setRecategorizationType(recategorization.getType()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); + } + } + } + + if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { + for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { + if (manualRemoval.getId().equals(redactionLogEntry.getId())) { + comments = manualRedactions.getComments().get(manualRemoval.getId()); + String manualOverrideReason = null; + if (manualRemoval.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setRedacted(false); + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); + } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); + redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + } + } + } + + if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { + for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { + if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualForceRedact.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setRedacted(true); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + + } + } + } + + if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { + for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { + if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); + redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + } + } + } + + + if (manualRedactions != null) { + comments = manualRedactions.getComments().get(redactionLogEntry.getId()); + } + + redactionLogEntry.setComments(comments); + } + + private String mergeReasonIfNecessary(String currentReason, String addition) { + if (currentReason != null) { + if (!currentReason.contains(addition)) { + return currentReason + addition; + } + return currentReason; + } else { + return ""; + } + } + + + public List addManualAddEntries(Set manualAdds, Map> comments, String dossierTemplateId) { + + List redactionLogEntries = new ArrayList<>(); + + for (ManualRedactionEntry manualRedactionEntry : manualAdds) { + + if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) { + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getId(), dossierTemplateId); + redactionLogEntry.setPositions(manualRedactionEntry.getPositions()); + redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId())); + } + } + + return redactionLogEntries; + } + + + private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { + + return manualRedactionEntry.getStatus().equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); + } + + + private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, + String dossierTemplateId) { + + return RedactionLogEntry.builder() + .id(id) + .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) + .reason(manualRedactionEntry.getReason()) + .isDictionaryEntry(manualRedactionEntry.isAddToDictionary()) + .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) + .legalBasis(manualRedactionEntry.getLegalBasis()) + .value(manualRedactionEntry.getValue()) + .type(manualRedactionEntry.getType()) + .redacted(true) + .isHint(false) + .section(null) + .sectionNumber(-1) + .manual(true) + .status(manualRedactionEntry.getStatus()) + .manualRedactionType(ManualRedactionType.ADD) + .isDictionaryEntry(false) + .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) + .build(); + } + + + private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction, boolean skipped) { + if (requested) { + return dictionaryService.getRequestRemoveColor(dossierTemplateId); + } + if ((!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) || skipped) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); + } + + + private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) { + + if (status.equals(Status.REQUESTED)) { + return dictionaryService.getRequestAddColor(dossierTemplateId); + } else if (status.equals(Status.DECLINED)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return getColor(type, dossierTemplateId); + } + + + private float[] getColor(String type, String dossierTemplateId) { + + return dictionaryService.getColor(type, dossierTemplateId); + } + + +} From d2ffb099876943aac4e2d47553f0e624cdc7c401 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 21 Jul 2021 16:01:16 +0300 Subject: [PATCH 24/46] redaction-log preview generation and removal of manual redactions from redaction-log --- .../v1/server/controller/RedactionController.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 047538cf..901104d0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -164,7 +164,13 @@ public class RedactionController implements RedactionResource { dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId()); var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); - return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); + + if (redactionLog.getComputationalVersion() == 0) { + // old redaction logs are returned directly + return redactionLog; + } else { + return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); + } } From 25e559475047beead9c067e070d3a8e4eafcacad Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 21 Jul 2021 17:25:50 +0300 Subject: [PATCH 25/46] fixed pmd --- .../v1/server/redaction/service/RedactionLogMergeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index ac19c236..6e368db7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -214,7 +214,7 @@ public class RedactionLogMergeService { if (requested) { return dictionaryService.getRequestRemoveColor(dossierTemplateId); } - if ((!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) || skipped) { + if (skipped || (!isRedaction && !dictionaryService.isHint(type, dossierTemplateId))) { return dictionaryService.getNotRedactedColor(dossierTemplateId); } return dictionaryService.getColor(type, dossierTemplateId); From 26e73121e9f3526c2fd68a3965f17adf74e28ede Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 21 Jul 2021 17:28:57 +0300 Subject: [PATCH 26/46] fixed pmd --- .../v1/server/redaction/service/RedactionLogMergeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 6e368db7..ba77eed7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -214,7 +214,7 @@ public class RedactionLogMergeService { if (requested) { return dictionaryService.getRequestRemoveColor(dossierTemplateId); } - if (skipped || (!isRedaction && !dictionaryService.isHint(type, dossierTemplateId))) { + if (skipped || !isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) { return dictionaryService.getNotRedactedColor(dossierTemplateId); } return dictionaryService.getColor(type, dossierTemplateId); From da14646cb7d8057c8189736565a024ef13921502 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 22 Jul 2021 18:52:36 +0300 Subject: [PATCH 27/46] some logs --- .../redaction/v1/server/controller/RedactionController.java | 1 + .../v1/server/redaction/service/RedactionLogMergeService.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 901104d0..7691239a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -165,6 +165,7 @@ public class RedactionController implements RedactionResource { var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); + log.info("Loaded redaction log with computationalVersion: {}",redactionLog.getComputationalVersion()); if (redactionLog.getComputationalVersion() == 0) { // old redaction logs are returned directly return redactionLog; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index ba77eed7..7f78e87a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import com.iqser.red.service.redaction.v1.model.*; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -9,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +@Slf4j @Service @RequiredArgsConstructor public class RedactionLogMergeService { @@ -18,6 +20,7 @@ public class RedactionLogMergeService { public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { + log.info("Merging Redaction log with manual redactions "); if (manualRedactions != null) { var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); From 00dd6b90081e7e99ac64a27d115a195d6ab0bc2a Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 22 Jul 2021 22:26:16 +0300 Subject: [PATCH 28/46] manual redaction fixes --- .../v1/server/redaction/service/RedactionLogMergeService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 7f78e87a..25d076cb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -20,13 +20,14 @@ public class RedactionLogMergeService { public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { - log.info("Merging Redaction log with manual redactions "); + log.info("Merging Redaction log with manual redactions {}", manualRedactions); if (manualRedactions != null) { var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); + log.info("Adding {}", manualRedactionLogEntries); for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { processRedactionLogEntry(manualRedactions, dossierTemplateId, entry); entry.setComments(manualRedactions.getComments().get(entry.getId())); @@ -175,6 +176,7 @@ public class RedactionLogMergeService { RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getId(), dossierTemplateId); redactionLogEntry.setPositions(manualRedactionEntry.getPositions()); redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId())); + redactionLogEntries.add(redactionLogEntry); } } From 0832ea15c09335cfb0d9f0139764ab6c99a22423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Mon, 26 Jul 2021 14:54:13 +0200 Subject: [PATCH 29/46] RED-1806: Use localDictionary also in Reanlysis to fix problem at removing values from false_positive that affect rules that should find values in entire document --- .../service/EntityRedactionService.java | 115 ++++++++------ .../redaction/service/ReanalyzeService.java | 145 ++++++++++-------- .../v1/server/RedactionIntegrationTest.java | 102 +++++++----- .../resources/dictionaries/CBI_author.txt | 1 - .../resources/dictionaries/false_positive.txt | 3 +- 5 files changed, 211 insertions(+), 155 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index ea1079a4..c1dc3f11 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -43,35 +43,13 @@ public class EntityRedactionService { if (dictionary.hasLocalEntries()) { - Map> hintsPerSectionNumber = new HashMap<>(); - documentEntities.stream().forEach(entity -> { - if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) { - hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()) - .add(entity); - } - }); - + Map> hintsPerSectionNumber = getHintsPerSection(documentEntities, dictionary); Set foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber, fileAttributes); EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary); EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities); } - for (Entity entity : documentEntities) { - Map> sequenceOnPage = new HashMap<>(); - for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { - sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()) - .add(entityPositionSequence); - } - - for (Map.Entry> entry : sequenceOnPage.entrySet()) { - classifiedDoc.getEntities() - .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) - .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry - .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity - .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity - .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry())); - } - } + classifiedDoc.setEntities(convertToEnititesPerPage(documentEntities)); dictionaryService.updateExternalDictionary(dictionary, dossierTemplateId); @@ -80,6 +58,39 @@ public class EntityRedactionService { } + public Map> convertToEnititesPerPage(Set entities){ + Map> entitiesPerPage = new HashMap<>(); + for (Entity entity : entities) { + Map> sequenceOnPage = new HashMap<>(); + for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { + sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()) + .add(entityPositionSequence); + } + + for (Map.Entry> entry : sequenceOnPage.entrySet()) { + entitiesPerPage + .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) + .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry + .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity + .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity + .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry())); + } + } + return entitiesPerPage; + } + + + public Map> getHintsPerSection(Set entities, Dictionary dictionary){ + Map> hintsPerSectionNumber = new HashMap<>(); + entities.stream().forEach(entity -> { + if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) { + hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()) + .add(entity); + } + }); + return hintsPerSectionNumber; + } + private Set findEntities(Document classifiedDoc, KieContainer kieContainer, ManualRedactions manualRedactions, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber, @@ -123,42 +134,46 @@ public class EntityRedactionService { } sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { - Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); - documentEntities.addAll(analysedRowSection.getEntities()); + Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); + documentEntities.addAll(analysedSection.getEntities()); - for (Image image : analysedRowSection.getImages()) { + for (Image image : analysedSection.getImages()) { classifiedDoc.getImages().computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image); } - analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> { - if (dictionary.isRecommendation(key)) { - analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> { - if (!dictionary.containsValue(key, value)) { - dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); - } - }); - } else { - analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> { - - if (dictionary.getLocalAccessMap().get(key) == null) { - log.warn("Dictionary {} is null", key); - } - - if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) { - log.warn("Dictionary {} localEntries is null", key); - } - - dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); - }); - } - }); - + addLocalValuesToDictionary(analysedSection, dictionary); }); return documentEntities; } + public void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary){ + analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> { + if (dictionary.isRecommendation(key)) { + analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> { + if (!dictionary.containsValue(key, value)) { + dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); + } + }); + } else { + analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> { + + if (dictionary.getLocalAccessMap().get(key) == null) { + log.warn("Dictionary {} is null", key); + } + + if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) { + log.warn("Dictionary {} localEntries is null", key); + } + + dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); + }); + } + }); + } + + private List processTablePerRow(Document classifiedDoc, Table table, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 17dfdf4b..810619a1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -13,9 +13,11 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; + import org.kie.api.runtime.KieContainer; import org.springframework.stereotype.Service; import org.springframework.web.bind.annotation.RequestBody; @@ -60,8 +62,7 @@ public class ReanalyzeService { entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest .getDossierId(), analyzeRequest.getFileAttributes()); - redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest - .getDossierTemplateId()); + redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getDossierTemplateId()); log.info("Redaction analysis successful..."); @@ -105,6 +106,59 @@ public class ReanalyzeService { DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); + Set sectionsToReanalyse = findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest); + + if (sectionsToReanalyse.isEmpty()) { + return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); + } + + List reanalysisSections = text.getSectionTexts() + .stream() + .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber())) + .collect(Collectors.toList()); + + KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); + + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest + .getDossierId()); + + Map> imagesPerPage = new HashMap<>(); + Set entities = findEntities(reanalysisSections, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage); + + if (dictionary.hasLocalEntries()) { + Map> hintsPerSectionNumber = entityRedactionService.getHintsPerSection(entities, dictionary); + Set foundByLocal = findEntities(reanalysisSections, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage); + EntitySearchUtils.addEntitiesWithHigherRank(entities, foundByLocal, dictionary); + EntitySearchUtils.removeEntitiesContainedInLarger(entities); + } + + Map> entitiesPerPage = entityRedactionService.convertToEnititesPerPage(entities); + + List newRedactionLogEntries = new ArrayList<>(); + for (int page = 1; page <= text.getNumberOfPages(); page++) { + if (entitiesPerPage.get(page) != null) { + newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, page, analyzeRequest + .getDossierTemplateId())); + } + + if (imagesPerPage.get(page) != null) { + newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, page, analyzeRequest + .getDossierTemplateId())); + } + + } + + redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); + redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); + AnalyzeResult analyzeResult = finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); + analyzeResult.setWasReanalyzed(true); + return analyzeResult; + } + + + private Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, + Text text, AnalyzeRequest analyzeRequest) { + Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set sectionsToReanalyse = new HashSet<>(); @@ -128,31 +182,20 @@ public class ReanalyzeService { log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest); - if (sectionsToReanalyse.isEmpty()) { - return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); - } + return sectionsToReanalyse; + } - List reanalysisSections = new ArrayList<>(); - for (SectionText sectionText : text.getSectionTexts()) { - - if (sectionsToReanalyse.contains(sectionText.getSectionNumber())) { - reanalysisSections.add(sectionText); - } - } - - //-- - - KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest - .getDossierId()); + private Set findEntities(List reanalysisSections, Dictionary dictionary, + KieContainer kieContainer, AnalyzeRequest analyzeRequest, boolean local, + Map> hintsPerSectionNumber, + Map> imagesPerPage) { List sectionSearchableTextPairs = new ArrayList<>(); for (SectionText reanalysisSection : reanalysisSections) { Set entities = entityRedactionService.findEntities(reanalysisSection.getSearchableText(), reanalysisSection - .getHeadline(), reanalysisSection.getSectionNumber(), dictionary, false); + .getHeadline(), reanalysisSection.getSectionNumber(), dictionary, local); if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) { surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection .getCellStarts()); @@ -160,14 +203,15 @@ public class ReanalyzeService { surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary); } - if (reanalysisSection.getImages() != null && !reanalysisSection.getImages() + if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages() .isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions() .getImageRecategorizations() != null) { for (Image image : reanalysisSection.getImages()) { String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() .getImageRecategorizations()) { - if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId().equals(imageId)) { + if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId() + .equals(imageId)) { image.setType(imageRecategorization.getType()); } } @@ -177,7 +221,10 @@ public class ReanalyzeService { sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() .isLocal(false) .dictionaryTypes(dictionary.getTypes()) - .entities(entities) + .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream + .concat(entities.stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()) + .stream()) + .collect(Collectors.toSet()) : entities) .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks()) .searchText(reanalysisSection.getSearchableText().toString()) .headline(reanalysisSection.getHeadline()) @@ -191,54 +238,19 @@ public class ReanalyzeService { } Set entities = new HashSet<>(); - Map> imagesPerPage = new HashMap<>(); sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { - Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); - entities.addAll(analysedRowSection.getEntities()); + Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); + entities.addAll(analysedSection.getEntities()); EntitySearchUtils.removeEntitiesContainedInLarger(entities); - for (Image image : analysedRowSection.getImages()) { + for (Image image : analysedSection.getImages()) { imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image); } + entityRedactionService.addLocalValuesToDictionary(analysedSection, dictionary); }); - Map> entitiesPerPage = new HashMap<>(); - for (Entity entity : entities) { - Map> sequenceOnPage = new HashMap<>(); - for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { - sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()) - .add(entityPositionSequence); - } - - for (Map.Entry> entry : sequenceOnPage.entrySet()) { - entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) - .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry - .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity - .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity - .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry())); - } - } - - List newRedactionLogEntries = new ArrayList<>(); - for (int page = 1; page <= text.getNumberOfPages(); page++) { - if (entitiesPerPage.get(page) != null) { - newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, page, analyzeRequest - .getDossierTemplateId())); - } - - if (imagesPerPage.get(page) != null) { - newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, page, analyzeRequest - .getDossierTemplateId())); - } - - } - - redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); - redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); - AnalyzeResult analyzeResult = finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); - analyzeResult.setWasReanalyzed(true); - return analyzeResult; + return entities; } @@ -296,10 +308,9 @@ public class ReanalyzeService { private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { - redactionLog.getRedactionLogEntry().forEach(entry -> - entry.getPositions().forEach(pos -> - entry.setExcluded(excludedPages != null && excludedPages.contains(pos.getPage())) - )); + redactionLog.getRedactionLogEntry() + .forEach(entry -> entry.getPositions() + .forEach(pos -> entry.setExcluded(excludedPages != null && excludedPages.contains(pos.getPage())))); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 8904ea6c..6949ff40 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -18,7 +18,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.service.StorageService; + import lombok.SneakyThrows; + import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; @@ -84,7 +86,6 @@ public class RedactionIntegrationTest { private static final String PII = "PII"; - @Autowired private RedactionController redactionController; @@ -127,6 +128,7 @@ public class RedactionIntegrationTest { private final Map rankTypeMap = new HashMap<>(); private final Colors colors = new Colors(); private final Map reanlysisVersions = new HashMap<>(); + private final Set deleted = new HashSet<>(); private final static String TEST_DOSSIER_TEMPLATE_ID = "123"; private final static String TEST_DOSSIER_ID = "123"; @@ -152,18 +154,20 @@ public class RedactionIntegrationTest { return kieServices.newKieContainer(kieModule.getReleaseId()); } + @Bean @Primary public StorageService inmemoryStorage() { + return new FileSystemBackedStorageService(); } - } @After public void cleanupStorage() { + if (this.storageService instanceof FileSystemBackedStorageService) { ((FileSystemBackedStorageService) this.storageService).clearStorage(); } @@ -179,7 +183,8 @@ public class RedactionIntegrationTest { loadDictionaryForTest(); loadTypeForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(0L); - when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse.builder() + when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse + .builder() .types(getTypeResponse()) .build()); @@ -188,7 +193,7 @@ public class RedactionIntegrationTest { .types(List.of(TypeResult.builder() .type(DOSSIER_REDACTIONS) .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .hexColor( "#ffe187") + .hexColor("#ffe187") .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) @@ -196,26 +201,42 @@ public class RedactionIntegrationTest { .build())) .build()); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); - when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(ADDRESS, false)); - when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(AUTHOR, false)); - when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SPONSOR, false)); - when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); - when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); - when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(HINT_ONLY, false)); - when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); - when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); - when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(TEST_METHOD, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(SPONSOR, false)); + when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(HINT_ONLY, false)); + when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(MUST_REDACT, false)); + when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); + when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(TEST_METHOD, false)); when(dictionaryClient.getDictionaryForType(PII, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PII, false)); - when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false)); - when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false)); - when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); - when(dictionaryClient.getDictionaryForType(PURITY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PURITY, false)); + when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); + when(dictionaryClient.getDictionaryForType(PURITY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(PURITY, false)); when(dictionaryClient.getDictionaryForType(IMAGE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(IMAGE, false)); when(dictionaryClient.getDictionaryForType(OCR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(OCR, false)); when(dictionaryClient.getDictionaryForType(LOGO, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(LOGO, false)); - when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SIGNATURE, false)); - when(dictionaryClient.getDictionaryForType(FORMULA, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FORMULA, false)); + when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(SIGNATURE, false)); + when(dictionaryClient.getDictionaryForType(FORMULA, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(FORMULA, false)); when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS, TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true)); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } @@ -477,7 +498,8 @@ public class RedactionIntegrationTest { return DictionaryResponse.builder() .hexColor(typeColorMap.get(type)) - .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type))) + .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary + .get(type))) .isHint(hintTypeMap.get(type)) .isCaseInsensitive(caseInSensitiveMap.get(type)) .isRecommendation(recommendationTypeMap.get(type)) @@ -490,7 +512,8 @@ public class RedactionIntegrationTest { List dictionaryEntries = new ArrayList<>(); entries.forEach(entry -> { - dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, false)); + dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, deleted + .contains(entry) ? true : false)); }); return dictionaryEntries; } @@ -498,6 +521,7 @@ public class RedactionIntegrationTest { @Test public void test270Rotated() { + AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf"); MemoryStats.printMemoryStats(); AnalyzeResult result = reanalyzeService.analyze(request); @@ -508,12 +532,14 @@ public class RedactionIntegrationTest { @Test @Ignore public void testLargeScannedFileOOM() { + AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf"); MemoryStats.printMemoryStats(); AnalyzeResult result = reanalyzeService.analyze(request); assertThat(result).isNotNull(); } + @Test public void testMergedImages() throws IOException { @@ -552,14 +578,13 @@ public class RedactionIntegrationTest { long rend = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (rend - rstart)); - long end = System.currentTimeMillis(); System.out.println("duration: " + (end - start)); - } + @Test @Ignore public void noExceptionShouldBeThrownForAnyFiles() throws IOException { @@ -635,7 +660,12 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); request.setExcludedPages(Set.of(1)); - request.setFileAttributes(List.of(FileAttribute.builder().id("fileAttributeId").label("Vertebrate Study").placeholder("{fileattributes.vertebrateStudy}").value("true").build())); + request.setFileAttributes(List.of(FileAttribute.builder() + .id("fileAttributeId") + .label("Vertebrate Study") + .placeholder("{fileattributes.vertebrateStudy}") + .value("true") + .build())); AnalyzeResult result = reanalyzeService.analyze(request); @@ -683,12 +713,18 @@ public class RedactionIntegrationTest { dictionary.get(AUTHOR).add("physical"); reanlysisVersions.put("physical", 2L); -// dictionary.get(VERTEBRATE).add("s-metolachlor"); -// reanlysisVersions.put("s-metolachlor", 3L); + deleted.add("David Chubb"); + + dictionary.get(FALSE_POSITIVE).add("David Chubb"); + reanlysisVersions.put("David Chubb", 3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); start = System.currentTimeMillis(); @@ -702,10 +738,8 @@ public class RedactionIntegrationTest { request.setManualRedactions(manualRedactions); - AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request); - redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); end = System.currentTimeMillis(); @@ -775,7 +809,6 @@ public class RedactionIntegrationTest { .status(Status.APPROVED) .build())); - manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment)); manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment)); manualRedactions.getComments().put(manualAddId, List.of(comment)); @@ -790,12 +823,10 @@ public class RedactionIntegrationTest { // manualRedactions.getEntriesToAdd().add(manualRedactionEntry); - AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); request.setManualRedactions(manualRedactions); AnalyzeResult result = reanalyzeService.analyze(request); - manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() .id("5b940b2cb401ed9f5be6fc24f6e77bcf") @@ -816,7 +847,6 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } @@ -833,7 +863,6 @@ public class RedactionIntegrationTest { System.out.println("classificationTest"); ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf"); - AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); RedactionRequest redactionRequest = RedactionRequest.builder() @@ -934,8 +963,10 @@ public class RedactionIntegrationTest { }); } + @SneakyThrows private AnalyzeRequest prepareStorage(String file) { + ClassPathResource pdfFileResource = new ClassPathResource(file); return prepareStorage(pdfFileResource.getInputStream()); @@ -967,7 +998,6 @@ public class RedactionIntegrationTest { long start = System.currentTimeMillis(); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf"); - AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); AnalyzeResult result = reanalyzeService.analyze(request); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt index 766ebd06..3d53e6d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt @@ -1676,7 +1676,6 @@ da Silva Rejane Das R Das, R. Daughtry, CST -David Chubb David Chubb|Lorraine Britton David Clarke Davies diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt index 0bb75f29..a696cee0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt @@ -235,4 +235,5 @@ N/A No details reported Not available Test facility -TBD \ No newline at end of file +TBD +David Chubb \ No newline at end of file From 289077e5e386e3a0785e2c849ae06bcb01202d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 29 Jul 2021 11:40:25 +0200 Subject: [PATCH 30/46] Fixed dossier dictionary and dictionary rank sorting --- .../v1/server/redaction/service/DictionaryService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index 5afff568..72d0d387 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -212,7 +212,7 @@ public class DictionaryService { dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); } - return new Dictionary(copy, DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); + return new Dictionary(copy.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); } From 77ece8b0d680326db8e7afee4270631f968b41b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 29 Jul 2021 13:15:07 +0200 Subject: [PATCH 31/46] Simplyfied hyphen removal regex --- .../v1/server/redaction/utils/TextNormalizationUtilities.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java index b740286b..342f6b03 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java @@ -12,7 +12,7 @@ public class TextNormalizationUtilities { * @return Text without line-break hyphenation. */ public static String removeHyphenLineBreaks(String text) { - return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2"); + return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R", "$1"); } } From cf12e58c77b30fd4ff790a3b038b3ba625aa5a28 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 29 Jul 2021 21:07:14 +0300 Subject: [PATCH 32/46] smart reanalysis based on sections to reanalyse provided by search service --- .../red/service/redaction/v1/model/AnalyzeRequest.java | 7 ++++++- .../v1/server/redaction/service/ReanalyzeService.java | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 4a7be09b..8f287ddd 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -7,6 +7,7 @@ import lombok.NoArgsConstructor; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -22,7 +23,11 @@ public class AnalyzeRequest { private boolean reanalyseOnlyIfPossible; private ManualRedactions manualRedactions; private OffsetDateTime lastProcessed; - private Set excludedPages; + + @Builder.Default + private Set excludedPages = new HashSet<>(); + @Builder.Default + private Set sectionsToReanalyse = new HashSet<>(); @Builder.Default private List fileAttributes = new ArrayList<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 810619a1..e2a19920 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -106,7 +106,8 @@ public class ReanalyzeService { DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set sectionsToReanalyse = findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest); + Set sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse().isEmpty() ? analyzeRequest.getSectionsToReanalyse() : + findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest); if (sectionsToReanalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); From af1fb030258257a3839439b57ec74733bd0e67dc Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 5 Aug 2021 10:36:59 +0300 Subject: [PATCH 33/46] fixed typo --- .../v1/server/redaction/service/RedactionLogMergeService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 25d076cb..52ca7f10 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -209,8 +209,6 @@ public class RedactionLogMergeService { .manual(true) .status(manualRedactionEntry.getStatus()) .manualRedactionType(ManualRedactionType.ADD) - .isDictionaryEntry(false) - .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) .build(); } From b08cdd3a07f763c6b310a3403c69d4b9652a0e5c Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 5 Aug 2021 10:59:01 +0300 Subject: [PATCH 34/46] cleaned up code, added analysisVersion for analyseresult and interogation, usefull later --- .../redaction/v1/model/AnalyzeResult.java | 2 ++ .../redaction/v1/model/RedactionLog.java | 2 +- .../v1/model/RedactionServiceDetails.java | 14 +++++++++++ .../v1/resources/RedactionResource.java | 6 ++--- .../controller/RedactionController.java | 25 ++++++++----------- .../service/AnalyzeResponseService.java | 7 ++++++ .../service/DroolsExecutionService.java | 4 +-- .../redaction/service/ReanalyzeService.java | 4 ++- .../settings/RedactionServiceSettings.java | 2 ++ 9 files changed, 44 insertions(+), 22 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionServiceDetails.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java index 5eadb70f..8d569c81 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java @@ -27,6 +27,8 @@ public class AnalyzeResult { private boolean wasReanalyzed; + private int analysisVersion; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index 276363f8..942e36a6 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -15,7 +15,7 @@ public class RedactionLog { * Version 0 Redaction Logs have manual redactions merged inside them * Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime. */ - private long computationalVersion; + private long analysisVersion; private List redactionLogEntry; private List legalBasis; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionServiceDetails.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionServiceDetails.java new file mode 100644 index 00000000..4bb34591 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionServiceDetails.java @@ -0,0 +1,14 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class RedactionServiceDetails { + + private int analysisVersion; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index 3abe15f5..18871367 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -24,13 +24,13 @@ public interface RedactionResource { @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest); - @PostMapping(value = "/rules/update" + RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE) - void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId); - @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE) void testRules(@RequestBody String rules); @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE) RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest); + @PostMapping(value = "/details", produces = MediaType.APPLICATION_JSON_VALUE) + RedactionServiceDetails getRedactionServiceDetails(); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 7691239a..33fa3ebf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -1,11 +1,7 @@ package com.iqser.red.service.redaction.v1.server.controller; import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.AnnotateRequest; -import com.iqser.red.service.redaction.v1.model.AnnotateResponse; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionRequest; -import com.iqser.red.service.redaction.v1.model.RedactionResult; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.resources.RedactionResource; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; @@ -15,6 +11,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.DictionarySer import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; @@ -42,6 +39,7 @@ public class RedactionController implements RedactionResource { private final PdfSegmentationService pdfSegmentationService; private final RedactionStorageService redactionStorageService; private final RedactionLogMergeService redactionLogMergeService; + private final RedactionServiceSettings redactionServiceSettings; public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { @@ -143,14 +141,6 @@ public class RedactionController implements RedactionResource { } - - @Override - public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId) { - - droolsExecutionService.updateRules(dossierTemplateId); - } - - @Override public void testRules(@RequestBody String rules) { @@ -165,8 +155,8 @@ public class RedactionController implements RedactionResource { var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); - log.info("Loaded redaction log with computationalVersion: {}",redactionLog.getComputationalVersion()); - if (redactionLog.getComputationalVersion() == 0) { + log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion()); + if (redactionLog.getAnalysisVersion() == 0) { // old redaction logs are returned directly return redactionLog; } else { @@ -174,6 +164,11 @@ public class RedactionController implements RedactionResource { } } + @Override + public RedactionServiceDetails getRedactionServiceDetails() { + return new RedactionServiceDetails(redactionServiceSettings.getAnalysisVersion()); + } + private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index ff772521..2ddbcc46 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -5,11 +5,17 @@ import com.iqser.red.service.redaction.v1.model.RedactionChangeLog; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @Service +@RequiredArgsConstructor public class AnalyzeResponseService { + private final RedactionServiceSettings redactionServiceSettings; + public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { @@ -51,6 +57,7 @@ public class AnalyzeResponseService { .hasRequests(hasRequests) .hasImages(hasImages) .hasUpdates(hasUpdates) + .analysisVersion(redactionServiceSettings.getAnalysisVersion()) .rulesVersion(redactionLog.getRulesVersion()) .dictionaryVersion(redactionLog.getDictionaryVersion()) .legalBasisVersion(redactionLog.getLegalBasisVersion()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index bd26cc12..0ac53bc6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -63,7 +63,7 @@ public class DroolsExecutionService { rulesVersion = -1L; } - if (version > rulesVersion.longValue()) { + if (version > rulesVersion) { rulesVersionPerDossierTemplateId.put(dossierTemplateId, version); return createOrUpdateKieContainer(dossierTemplateId); } @@ -128,7 +128,7 @@ public class DroolsExecutionService { if (rulesVersion == null) { return -1; } - return rulesVersion.longValue(); + return rulesVersion; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index e2a19920..e3b37c5f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -12,6 +12,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import lombok.RequiredArgsConstructor; @@ -41,6 +42,7 @@ public class ReanalyzeService { private final RedactionChangeLogService redactionChangeLogService; private final AnalyzeResponseService analyzeResponseService; private final LegalBasisClient legalBasisClient; + private final RedactionServiceSettings redactionServiceSettings; public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { @@ -67,7 +69,7 @@ public class ReanalyzeService { log.info("Redaction analysis successful..."); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - var redactionLog = new RedactionLog(1, classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() + var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() .getDossierTemplateVersion(), classifiedDoc.getDictionaryVersion() .getDossierVersion(), classifiedDoc.getRulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java index 3d5b0b5b..dfc34079 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java @@ -15,4 +15,6 @@ public class RedactionServiceSettings { private float maxImageCropboxRatio = 0.9f; + private int analysisVersion = 1; + } From 4bdca5644a7296bd542278ffa2f64e670855fbe5 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 5 Aug 2021 13:13:48 +0300 Subject: [PATCH 35/46] cleanup --- .../v1/model/RedactionServiceDetails.java | 14 -------------- .../redaction/v1/resources/RedactionResource.java | 7 ------- .../v1/server/controller/RedactionController.java | 7 ------- 3 files changed, 28 deletions(-) delete mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionServiceDetails.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionServiceDetails.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionServiceDetails.java deleted file mode 100644 index 4bb34591..00000000 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionServiceDetails.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.iqser.red.service.redaction.v1.model; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class RedactionServiceDetails { - - private int analysisVersion; - -} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index 18871367..d6bb66f9 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -8,10 +8,6 @@ import org.springframework.web.bind.annotation.RequestBody; public interface RedactionResource { - String RULE_SET_PARAMETER_NAME = "dossierTemplateId"; - String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}"; - - @PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest); @@ -30,7 +26,4 @@ public interface RedactionResource { @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE) RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest); - @PostMapping(value = "/details", produces = MediaType.APPLICATION_JSON_VALUE) - RedactionServiceDetails getRedactionServiceDetails(); - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 33fa3ebf..95815664 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -39,7 +39,6 @@ public class RedactionController implements RedactionResource { private final PdfSegmentationService pdfSegmentationService; private final RedactionStorageService redactionStorageService; private final RedactionLogMergeService redactionLogMergeService; - private final RedactionServiceSettings redactionServiceSettings; public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { @@ -164,12 +163,6 @@ public class RedactionController implements RedactionResource { } } - @Override - public RedactionServiceDetails getRedactionServiceDetails() { - return new RedactionServiceDetails(redactionServiceSettings.getAnalysisVersion()); - } - - private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException { try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { From 30715cdcb85e94911c3903fe95b1cb902ed6556f Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 5 Aug 2021 15:35:06 +0300 Subject: [PATCH 36/46] added user-id to all manual entries --- .../red/service/redaction/v1/model/RedactionLogEntry.java | 1 + .../server/redaction/service/RedactionLogMergeService.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index 9f7a0146..38497fa5 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -36,6 +36,7 @@ public class RedactionLogEntry { private boolean manual; private Status status; private ManualRedactionType manualRedactionType; + private String manualRedactionUserId; private boolean isDictionaryEntry; private String textBefore; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 52ca7f10..cade6618 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -61,6 +61,7 @@ public class RedactionLogMergeService { redactionLogEntry.setStatus(Status.DECLINED); } + redactionLogEntry.setManualRedactionUserId(recategorization.getUser()); redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setManual(true); redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); @@ -88,6 +89,7 @@ public class RedactionLogMergeService { redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser()); redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); @@ -114,6 +116,7 @@ public class RedactionLogMergeService { redactionLogEntry.setStatus(Status.DECLINED); } + redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser()); redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setManual(true); redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); @@ -139,6 +142,7 @@ public class RedactionLogMergeService { redactionLogEntry.setStatus(Status.DECLINED); } + redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser()); redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setManual(true); redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); @@ -209,6 +213,7 @@ public class RedactionLogMergeService { .manual(true) .status(manualRedactionEntry.getStatus()) .manualRedactionType(ManualRedactionType.ADD) + .manualRedactionUserId(manualRedactionEntry.getUser()) .build(); } From e8a4ef172c994ab8dd99f84f7a725e3dddfaca5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Wed, 11 Aug 2021 09:45:00 +0200 Subject: [PATCH 37/46] RED-1908: Removed RedactionChangeLog added changes to RedactionLog --- .../service/redaction/v1/model/Change.java | 18 +++ .../redaction/v1/model/ChangeType.java | 2 +- .../v1/model/RedactionChangeLog.java | 22 --- .../v1/model/RedactionChangeLogEntry.java | 49 ------ .../v1/model/RedactionLogChanges.java | 17 ++ .../redaction/v1/model/RedactionLogEntry.java | 4 + .../service/AnalyzeResponseService.java | 12 +- .../redaction/service/ReanalyzeService.java | 14 +- .../service/RedactionChangeLogService.java | 145 +++++++++--------- .../v1/server/RedactionIntegrationTest.java | 17 ++ 10 files changed, 142 insertions(+), 158 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java delete mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java delete mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java new file mode 100644 index 00000000..ebc59a84 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.model; + +import java.time.OffsetDateTime; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class Change { + + private ChangeType type; + private OffsetDateTime dateTime; +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java index 0c902a8f..53438d3c 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java @@ -1,5 +1,5 @@ package com.iqser.red.service.redaction.v1.model; public enum ChangeType { - ADDED, REMOVED + ADDED, REMOVED, CHANGED } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java deleted file mode 100644 index 74e385c0..00000000 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.iqser.red.service.redaction.v1.model; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.ArrayList; -import java.util.List; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class RedactionChangeLog { - - private List redactionLogEntry = new ArrayList<>(); - - private long dictionaryVersion = -1; - private long dossierDictionaryVersion = -1; - private long rulesVersion = -1; - private long legalBasisVersion = -1; - -} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java deleted file mode 100644 index a53d3b0e..00000000 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.iqser.red.service.redaction.v1.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.ArrayList; -import java.util.List; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class RedactionChangeLogEntry { - - private String id; - private String type; - private String value; - private String reason; - private int matchedRule; - private String legalBasis; - private boolean redacted; - private boolean isHint; - private boolean isRecommendation; - private String section; - private float[] color; - - @Builder.Default - private List positions = new ArrayList<>(); - private int sectionNumber; - private boolean manual; - private Status status; - private ManualRedactionType manualRedactionType; - private boolean isDictionaryEntry; - - private String textBefore; - private String textAfter; - - @Builder.Default - private List comments = new ArrayList<>(); - - private ChangeType changeType; - - private boolean isDossierDictionaryEntry; - - private boolean excluded; - -} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java new file mode 100644 index 00000000..77fe374d --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class RedactionLogChanges { + + private RedactionLog redactionLog; + private boolean hasChanges; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index 38497fa5..dc14eebc 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -58,4 +58,8 @@ public class RedactionLogEntry { private String recategorizationType; private String legalBasisChangeValue; + @EqualsAndHashCode.Exclude + @Builder.Default + private List changes = new ArrayList<>(); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index 2ddbcc46..bd8042ff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -1,12 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLog; import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; -import lombok.NoArgsConstructor; + import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -17,7 +15,7 @@ public class AnalyzeResponseService { private final RedactionServiceSettings redactionServiceSettings; public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, - RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { + RedactionLog redactionLog, boolean hasUpdates) { boolean hasHints = redactionLog.getRedactionLogEntry() .stream() @@ -41,12 +39,6 @@ public class AnalyzeResponseService { .filter(entry -> !entry.isExcluded()) .anyMatch(entry -> entry.isHint() && entry.getType().equals("image") || entry.isImage()); - boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog - .getRedactionLogEntry() - .isEmpty() && redactionChangeLog.getRedactionLogEntry() - .stream() - .anyMatch(entry -> !entry.getType().equals("false_positive")); - return AnalyzeResult.builder() .dossierId(dossierId) .fileId(fileId) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index e3b37c5f..21df6b0f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -78,9 +78,9 @@ public class ReanalyzeService { log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc .getDictionaryVersion(), analyzeRequest.getDossierTemplateId()); - // first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data - var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); - // store redactionLog + var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); + redactionLog = redactionLogChange.getRedactionLog(); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc .getSectionText())); @@ -88,7 +88,7 @@ public class ReanalyzeService { .getSectionGrid()); long duration = System.currentTimeMillis() - startTime; - return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog); + return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, redactionLogChange.isHasChanges()); } @@ -266,13 +266,13 @@ public class ReanalyzeService { excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); - var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); + var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); long duration = System.currentTimeMillis() - startTime; return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text - .getNumberOfPages(), redactionLog, changeLog); + .getNumberOfPages(), redactionLogChange.getRedactionLog(), redactionLogChange.isHasChanges()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index 1cc743e5..41772291 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -1,19 +1,25 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.ChangeType; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLog; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLogEntry; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; +import com.iqser.red.service.redaction.v1.model.Change; +import com.iqser.red.service.redaction.v1.model.ChangeType; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogChanges; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @@ -22,76 +28,77 @@ public class RedactionChangeLogService { private final RedactionStorageService redactionStorageService; - public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) { - try { - RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId); - var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog); - redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog); - return changeLog; - } catch (Exception e) { - log.debug("Previous redaction log not available"); - return null; - } - - } - - - private RedactionChangeLog createChangeLog(RedactionLog currentRedactionLog, RedactionLog previousRedactionLog) { + public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) { + RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId); if (previousRedactionLog == null) { - return null; + currentRedactionLog.getRedactionLogEntry().forEach(entry -> { + entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + }); + return new RedactionLogChanges(currentRedactionLog, false); } - List added = new ArrayList<>(currentRedactionLog.getRedactionLogEntry()); - added.removeAll(previousRedactionLog.getRedactionLogEntry()); + List notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> !entry.getChanges() + .get(entry.getChanges().size() - 1) + .getType() + .equals(ChangeType.REMOVED)) + .collect(Collectors.toList()); - List removed = new ArrayList<>(previousRedactionLog.getRedactionLogEntry()); + Set added = new HashSet<>(currentRedactionLog.getRedactionLogEntry()); + added.removeAll(notRemovedPreviousEntries); + + Set removed = new HashSet<>(notRemovedPreviousEntries); removed.removeAll(currentRedactionLog.getRedactionLogEntry()); - List changeLogEntries = added.stream() - .map(entry -> convert(entry, ChangeType.ADDED)) - .collect(Collectors.toList()); - changeLogEntries.addAll(removed.stream() - .map(entry -> convert(entry, ChangeType.REMOVED)) - .collect(Collectors.toList())); + Map addedIds = new HashMap<>(); + added.forEach(entry -> { + addedIds.put(entry.getId(), entry); + }); - return new RedactionChangeLog(changeLogEntries, - currentRedactionLog.getDictionaryVersion(), - currentRedactionLog.getDossierDictionaryVersion(), - currentRedactionLog.getRulesVersion(), - currentRedactionLog.getLegalBasisVersion()); - } + Set removedIds = new HashSet<>(); + removed.forEach(entry -> { + removedIds.add(entry.getId()); + }); + List newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry(); - private RedactionChangeLogEntry convert(RedactionLogEntry entry, ChangeType changeType) { + List toRemove = new ArrayList<>(); + newRedactionLogEntries.forEach(entry -> { + if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now())); + var newEntry = addedIds.get(entry.getId()); + newEntry.setChanges(changes); + addedIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } else if (removedIds.contains(entry.getId())) { + entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now())); + } else if (addedIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + var newEntry = addedIds.get(entry.getId()); + newEntry.setChanges(changes); + addedIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } + }); - return RedactionChangeLogEntry.builder() - .id(entry.getId()) - .type(entry.getType()) - .value(entry.getValue()) - .reason(entry.getReason()) - .matchedRule(entry.getMatchedRule()) - .legalBasis(entry.getLegalBasis()) - .redacted(entry.isRedacted()) - .isHint(entry.isHint()) - .isRecommendation(entry.isRecommendation()) - .section(entry.getSection()) - .color(entry.getColor()) - .positions(entry.getPositions()) - .sectionNumber(entry.getSectionNumber()) - .manual(entry.isManual()) - .status(entry.getStatus()) - .manualRedactionType(entry.getManualRedactionType()) - .isDictionaryEntry(entry.isDictionaryEntry()) - .textBefore(entry.getTextBefore()) - .textAfter(entry.getTextAfter()) - .comments(entry.getComments()) - .changeType(changeType) - .isDossierDictionaryEntry(entry.isDossierDictionaryEntry()) - .excluded(entry.isExcluded()) - .build(); + newRedactionLogEntries.removeAll(toRemove); + + addedIds.forEach((k, v) -> { + if(v.getChanges().isEmpty()) { + v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + } + newRedactionLogEntries.add(v); + }); + + currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries); + + return new RedactionLogChanges(currentRedactionLog, !addedIds.isEmpty() || !removedIds.isEmpty()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 6949ff40..5a4daeb8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -714,10 +714,13 @@ public class RedactionIntegrationTest { reanlysisVersions.put("physical", 2L); deleted.add("David Chubb"); + deleted.add("mouse"); dictionary.get(FALSE_POSITIVE).add("David Chubb"); reanlysisVersions.put("David Chubb", 3L); + reanlysisVersions.put("mouse", 3L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) @@ -754,6 +757,20 @@ public class RedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } + + deleted.remove("mouse"); + reanlysisVersions.put("mouse", 4L); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L); + + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + reanalyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + System.out.println("hi"); } From bb80b7ae7550d70a98b4d500d3870a13585b57e8 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 13 Aug 2021 11:19:54 +0300 Subject: [PATCH 38/46] Proper Order of manual actions based on time --- .../service/RedactionLogMergeService.java | 268 ++++++++++-------- 1 file changed, 153 insertions(+), 115 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index cade6618..8d1b734a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -1,14 +1,17 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import com.iqser.red.service.file.management.v1.api.model.manual.ForceRedactionRequest; import com.iqser.red.service.redaction.v1.model.*; +import lombok.AllArgsConstructor; +import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.kie.api.definition.rule.All; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; @Slf4j @Service @@ -27,9 +30,13 @@ public class RedactionLogMergeService { redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); + var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions); + log.info("Adding {}", manualRedactionLogEntries); for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - processRedactionLogEntry(manualRedactions, dossierTemplateId, entry); + + processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())) + .collect(Collectors.toList()), dossierTemplateId, entry); entry.setComments(manualRedactions.getComments().get(entry.getId())); } @@ -39,123 +46,140 @@ public class RedactionLogMergeService { } - private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry) { - - List comments = null; - - - if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { - for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { - if (recategorization.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (recategorization.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setType(recategorization.getType()); - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); - } else if (recategorization.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); - redactionLogEntry.setRecategorizationType(recategorization.getType()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - redactionLogEntry.setManualRedactionUserId(recategorization.getUser()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); - } + private List createManualRedactionWrappers(ManualRedactions manualRedactions) { + List manualRedactionWrappers = new ArrayList<>(); + manualRedactions.getForceRedacts().forEach(item -> { + if (item.getSoftDeletedTime() != null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } - } - - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(redactionLogEntry.getId())) { - comments = manualRedactions.getComments().get(manualRemoval.getId()); - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser()); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - } + }); + manualRedactions.getEntriesToAdd().forEach(item -> { + if (item.getSoftDeletedTime() != null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - - } + }); + manualRedactions.getIdsToRemove().forEach(item -> { + if (item.getSoftDeletedTime() != null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } - } - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } + }); + manualRedactions.getManualLegalBasisChanges().forEach(item -> { + if (item.getSoftDeletedTime() != null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } - } + }); + manualRedactions.getImageRecategorizations().forEach(item -> { + if (item.getSoftDeletedTime() != null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); + } + }); - if (manualRedactions != null) { - comments = manualRedactions.getComments().get(redactionLogEntry.getId()); - } + Collections.sort(manualRedactionWrappers); + return manualRedactionWrappers; + } + + private void processRedactionLogEntry(List manualRedactionWrappers, String dossierTemplateId, RedactionLogEntry redactionLogEntry) { + + + manualRedactionWrappers.forEach(mrw -> { + + if (mrw.getItem() instanceof ManualImageRecategorization) { + var imageRecategorization = (ManualImageRecategorization) mrw.getItem(); + String manualOverrideReason = null; + if (imageRecategorization.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setType(imageRecategorization.getType()); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); + } else if (imageRecategorization.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setRecategorizationType(imageRecategorization.getType()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); + } + + if (mrw.getItem() instanceof IdRemoval) { + var manualRemoval = (IdRemoval) mrw.getItem(); + String manualOverrideReason = null; + if (manualRemoval.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setRedacted(false); + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); + } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser()); + redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); + redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + } + + + if (mrw.getItem() instanceof ManualForceRedact) { + var manualForceRedact = (ManualForceRedact) mrw.getItem(); + String manualOverrideReason = null; + if (manualForceRedact.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setRedacted(true); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + + } + + if (mrw.getItem() instanceof ManualLegalBasisChange) { + var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem(); + String manualOverrideReason = null; + if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); + redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + } + + }); - redactionLogEntry.setComments(comments); } private String mergeReasonIfNecessary(String currentReason, String addition) { @@ -245,5 +269,19 @@ public class RedactionLogMergeService { return dictionaryService.getColor(type, dossierTemplateId); } + @Data + @AllArgsConstructor + private static class ManualRedactionWrapper implements Comparable { + + private String id; + private OffsetDateTime date; + private Object item; + + @Override + public int compareTo(ManualRedactionWrapper o) { + return this.date.compareTo(o.date); + } + } + } From e43e8bccbf283ccb5ea1eeb405020229fb66ea3c Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 13 Aug 2021 13:00:32 +0200 Subject: [PATCH 39/46] RedactionLogMergeService.java edited online with Bitbucket --- .../service/RedactionLogMergeService.java | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 8d1b734a..ce09f06c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -48,34 +48,36 @@ public class RedactionLogMergeService { private List createManualRedactionWrappers(ManualRedactions manualRedactions) { List manualRedactionWrappers = new ArrayList<>(); - manualRedactions.getForceRedacts().forEach(item -> { - if (item.getSoftDeletedTime() != null) { - manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); - } - }); - manualRedactions.getEntriesToAdd().forEach(item -> { - if (item.getSoftDeletedTime() != null) { - manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); - } - }); - manualRedactions.getIdsToRemove().forEach(item -> { - if (item.getSoftDeletedTime() != null) { - manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); - } - }); - manualRedactions.getManualLegalBasisChanges().forEach(item -> { - if (item.getSoftDeletedTime() != null) { - manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); - } - }); + + log.info("Processing Manual Redactions: {}", manualRedactions); + manualRedactions.getImageRecategorizations().forEach(item -> { - if (item.getSoftDeletedTime() != null) { + if (item.getSoftDeletedTime() == null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); + } + }); + + manualRedactions.getIdsToRemove().forEach(item -> { + if (item.getSoftDeletedTime() == null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); + } + }); + + manualRedactions.getForceRedacts().forEach(item -> { + if (item.getSoftDeletedTime() == null) { + manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); + } + }); + + manualRedactions.getManualLegalBasisChanges().forEach(item -> { + if (item.getSoftDeletedTime() == null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); Collections.sort(manualRedactionWrappers); + log.info("Obtained manual redaction wrappers: {}", manualRedactionWrappers); return manualRedactionWrappers; } From 5312d5ffdfcd8ab11d87815f4a90745f68d7ca29 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 13 Aug 2021 13:19:42 +0200 Subject: [PATCH 40/46] RedactionLogMergeService.java edited online with Bitbucket --- .../service/RedactionLogMergeService.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index ce09f06c..406d01e8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -37,6 +37,7 @@ public class RedactionLogMergeService { processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())) .collect(Collectors.toList()), dossierTemplateId, entry); + entry.setComments(manualRedactions.getComments().get(entry.getId())); } @@ -47,30 +48,30 @@ public class RedactionLogMergeService { private List createManualRedactionWrappers(ManualRedactions manualRedactions) { + log.info("Processing Manual Redactions: {}", manualRedactions); + List manualRedactionWrappers = new ArrayList<>(); - log.info("Processing Manual Redactions: {}", manualRedactions); - manualRedactions.getImageRecategorizations().forEach(item -> { - if (item.getSoftDeletedTime() == null) { + if (item.getSoftDeletedTime() != null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); manualRedactions.getIdsToRemove().forEach(item -> { - if (item.getSoftDeletedTime() == null) { + if (item.getSoftDeletedTime() != null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); manualRedactions.getForceRedacts().forEach(item -> { - if (item.getSoftDeletedTime() == null) { + if (item.getSoftDeletedTime() != null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); manualRedactions.getManualLegalBasisChanges().forEach(item -> { - if (item.getSoftDeletedTime() == null) { + if (item.getSoftDeletedTime() != null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); @@ -84,6 +85,8 @@ public class RedactionLogMergeService { private void processRedactionLogEntry(List manualRedactionWrappers, String dossierTemplateId, RedactionLogEntry redactionLogEntry) { + log.info("Processing: {} for: {}", manualRedactionWrappers, redactionLogEntry); + manualRedactionWrappers.forEach(mrw -> { if (mrw.getItem() instanceof ManualImageRecategorization) { @@ -287,3 +290,4 @@ public class RedactionLogMergeService { } + From 1f12f9571f582032a5e10386a00525a22b9a131c Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 13 Aug 2021 13:56:13 +0200 Subject: [PATCH 41/46] RedactionLogMergeService.java edited online with Bitbucket --- .../service/RedactionLogMergeService.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 406d01e8..86b90b7f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -32,7 +32,6 @@ public class RedactionLogMergeService { var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions); - log.info("Adding {}", manualRedactionLogEntries); for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())) @@ -48,44 +47,44 @@ public class RedactionLogMergeService { private List createManualRedactionWrappers(ManualRedactions manualRedactions) { - log.info("Processing Manual Redactions: {}", manualRedactions); + List manualRedactionWrappers = new ArrayList<>(); manualRedactions.getImageRecategorizations().forEach(item -> { - if (item.getSoftDeletedTime() != null) { + if (item.getSoftDeletedTime() == null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); manualRedactions.getIdsToRemove().forEach(item -> { - if (item.getSoftDeletedTime() != null) { + if (item.getSoftDeletedTime() == null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); manualRedactions.getForceRedacts().forEach(item -> { - if (item.getSoftDeletedTime() != null) { + if (item.getSoftDeletedTime() == null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); manualRedactions.getManualLegalBasisChanges().forEach(item -> { - if (item.getSoftDeletedTime() != null) { + if (item.getSoftDeletedTime() == null) { manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); } }); Collections.sort(manualRedactionWrappers); - log.info("Obtained manual redaction wrappers: {}", manualRedactionWrappers); + return manualRedactionWrappers; } private void processRedactionLogEntry(List manualRedactionWrappers, String dossierTemplateId, RedactionLogEntry redactionLogEntry) { - log.info("Processing: {} for: {}", manualRedactionWrappers, redactionLogEntry); + manualRedactionWrappers.forEach(mrw -> { From 4d514d0e5ed1c22436bd8ba58578022d5fb8c4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Tue, 17 Aug 2021 11:28:52 +0200 Subject: [PATCH 42/46] RED-2015: Fixed (Manual) Redactions in download reports/files for excluded pages --- .../redaction/v1/model/RedactionRequest.java | 5 + .../controller/RedactionController.java | 2 +- .../redaction/service/ReanalyzeService.java | 8 +- .../service/RedactionLogMergeService.java | 92 +++++++++++++------ 4 files changed, 75 insertions(+), 32 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java index 263e7692..d92f0edf 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java @@ -1,5 +1,8 @@ package com.iqser.red.service.redaction.v1.model; +import java.util.HashSet; +import java.util.Set; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -15,4 +18,6 @@ public class RedactionRequest { private String fileId; private String dossierTemplateId; private ManualRedactions manualRedactions; + @Builder.Default + private Set excludedPages = new HashSet<>(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 95815664..2ea691d7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -159,7 +159,7 @@ public class RedactionController implements RedactionResource { // old redaction logs are returned directly return redactionLog; } else { - return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); + return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 21df6b0f..1daf766b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -311,9 +311,11 @@ public class ReanalyzeService { private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { - redactionLog.getRedactionLogEntry() - .forEach(entry -> entry.getPositions() - .forEach(pos -> entry.setExcluded(excludedPages != null && excludedPages.contains(pos.getPage())))); + if(excludedPages != null && !excludedPages.isEmpty()) { + redactionLog.getRedactionLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { if (excludedPages.contains(pos.getPage())) { + entry.setExcluded(true); + }})); + } } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 86b90b7f..5c1d9fcc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -1,17 +1,31 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.file.management.v1.api.model.manual.ForceRedactionRequest; -import com.iqser.red.service.redaction.v1.model.*; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.model.Comment; +import com.iqser.red.service.redaction.v1.model.IdRemoval; +import com.iqser.red.service.redaction.v1.model.ManualForceRedact; +import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; +import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange; +import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; +import com.iqser.red.service.redaction.v1.model.ManualRedactionType; +import com.iqser.red.service.redaction.v1.model.ManualRedactions; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.Status; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.kie.api.definition.rule.All; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.util.*; -import java.util.stream.Collectors; @Slf4j @Service @@ -21,7 +35,8 @@ public class RedactionLogMergeService { private final DictionaryService dictionaryService; - public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { + public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, + ManualRedactions manualRedactions, Set excludedPages) { log.info("Merging Redaction log with manual redactions {}", manualRedactions); if (manualRedactions != null) { @@ -34,10 +49,19 @@ public class RedactionLogMergeService { for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())) + processRedactionLogEntry(manualRedactionWrappers.stream() + .filter(mr -> entry.getId().equals(mr.getId())) .collect(Collectors.toList()), dossierTemplateId, entry); entry.setComments(manualRedactions.getComments().get(entry.getId())); + + if (excludedPages != null && !excludedPages.isEmpty()) { + entry.getPositions().forEach(pos -> { + if (excludedPages.contains(pos.getPage())) { + entry.setExcluded(true); + } + }); + } } } @@ -47,8 +71,7 @@ public class RedactionLogMergeService { private List createManualRedactionWrappers(ManualRedactions manualRedactions) { - - + List manualRedactionWrappers = new ArrayList<>(); manualRedactions.getImageRecategorizations().forEach(item -> { @@ -75,16 +98,14 @@ public class RedactionLogMergeService { } }); - Collections.sort(manualRedactionWrappers); return manualRedactionWrappers; } - private void processRedactionLogEntry(List manualRedactionWrappers, String dossierTemplateId, RedactionLogEntry redactionLogEntry) { - - + private void processRedactionLogEntry(List manualRedactionWrappers, + String dossierTemplateId, RedactionLogEntry redactionLogEntry) { manualRedactionWrappers.forEach(mrw -> { @@ -98,7 +119,8 @@ public class RedactionLogMergeService { } else if (imageRecategorization.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry + .isRedacted(), false)); redactionLogEntry.setRecategorizationType(imageRecategorization.getType()); } else { redactionLogEntry.setStatus(Status.DECLINED); @@ -117,11 +139,13 @@ public class RedactionLogMergeService { redactionLogEntry.setRedacted(false); redactionLogEntry.setStatus(Status.APPROVED); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry + .isRedacted(), true)); } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry + .isRedacted(), false)); } else { redactionLogEntry.setStatus(Status.DECLINED); } @@ -134,20 +158,21 @@ public class RedactionLogMergeService { redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); } - if (mrw.getItem() instanceof ManualForceRedact) { var manualForceRedact = (ManualForceRedact) mrw.getItem(); String manualOverrideReason = null; if (manualForceRedact.getStatus().equals(Status.APPROVED)) { redactionLogEntry.setRedacted(true); redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry + .isRedacted(), false)); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry + .isRedacted(), false)); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else { redactionLogEntry.setStatus(Status.DECLINED); @@ -170,7 +195,8 @@ public class RedactionLogMergeService { } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry + .isRedacted(), false)); redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); } else { redactionLogEntry.setStatus(Status.DECLINED); @@ -186,7 +212,9 @@ public class RedactionLogMergeService { } + private String mergeReasonIfNecessary(String currentReason, String addition) { + if (currentReason != null) { if (!currentReason.contains(addition)) { return currentReason + addition; @@ -198,14 +226,16 @@ public class RedactionLogMergeService { } - public List addManualAddEntries(Set manualAdds, Map> comments, String dossierTemplateId) { + public List addManualAddEntries(Set manualAdds, + Map> comments, String dossierTemplateId) { List redactionLogEntries = new ArrayList<>(); for (ManualRedactionEntry manualRedactionEntry : manualAdds) { if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) { - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getId(), dossierTemplateId); + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry + .getId(), dossierTemplateId); redactionLogEntry.setPositions(manualRedactionEntry.getPositions()); redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId())); redactionLogEntries.add(redactionLogEntry); @@ -218,7 +248,8 @@ public class RedactionLogMergeService { private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { - return manualRedactionEntry.getStatus().equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); + return manualRedactionEntry.getStatus() + .equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); } @@ -246,7 +277,9 @@ public class RedactionLogMergeService { } - private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction, boolean skipped) { + private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction, + boolean skipped) { + if (requested) { return dictionaryService.getRequestRemoveColor(dossierTemplateId); } @@ -273,6 +306,7 @@ public class RedactionLogMergeService { return dictionaryService.getColor(type, dossierTemplateId); } + @Data @AllArgsConstructor private static class ManualRedactionWrapper implements Comparable { @@ -281,12 +315,14 @@ public class RedactionLogMergeService { private OffsetDateTime date; private Object item; + @Override public int compareTo(ManualRedactionWrapper o) { + return this.date.compareTo(o.date); } - } + } } From 078dc2c3c3508a2407a527ff3512239998a30869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Wed, 18 Aug 2021 10:02:37 +0200 Subject: [PATCH 43/46] RED-1755: Calculate falgs in file-management-service async --- .../redaction/v1/model/AnalyzeResult.java | 4 --- .../service/AnalyzeResponseService.java | 25 ------------------- 2 files changed, 29 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java index 8d569c81..4fc3fada 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java @@ -15,10 +15,6 @@ public class AnalyzeResult { private String fileId; private long duration; private int numberOfPages; - private boolean hasHints; - private boolean hasRequests; - private boolean hasRedactions; - private boolean hasImages; private boolean hasUpdates; private long dictionaryVersion; private long dossierDictionaryVersion; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index bd8042ff..63fbadf6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -17,37 +17,12 @@ public class AnalyzeResponseService { public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, RedactionLog redactionLog, boolean hasUpdates) { - boolean hasHints = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isHint() && !entry.getType().equals("false_positive")); - - boolean hasRequests = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isManual() && entry.getStatus() - .equals(com.iqser.red.service.redaction.v1.model.Status.REQUESTED)); - - boolean hasRedactions = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isRedacted() && !entry.isManual() || entry.isManual() && entry.getStatus() - .equals(com.iqser.red.service.redaction.v1.model.Status.APPROVED)); - - boolean hasImages = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isHint() && entry.getType().equals("image") || entry.isImage()); return AnalyzeResult.builder() .dossierId(dossierId) .fileId(fileId) .duration(duration) .numberOfPages(pageCount) - .hasHints(hasHints) - .hasRedactions(hasRedactions) - .hasRequests(hasRequests) - .hasImages(hasImages) .hasUpdates(hasUpdates) .analysisVersion(redactionServiceSettings.getAnalysisVersion()) .rulesVersion(redactionLog.getRulesVersion()) From bcd469bb657f003813b09fd98bdbb2f624aa5e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Fri, 20 Aug 2021 14:51:22 +0200 Subject: [PATCH 44/46] Added equalsIgnoreCase fileattributes methods for rules --- .../v1/server/redaction/model/Section.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index dc878a20..17209efb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -72,6 +72,19 @@ public class Section { } + public boolean fileAttributeByIdEqualsIgnoreCase(String id, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + } + + public boolean fileAttributeByPlaceholderEqualsIgnoreCase(String placeholder, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + } + + public boolean fileAttributeByLabelEqualsIgnoreCase(String label, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + } + + public boolean rowEquals(String headerName, String value) { String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); From 76bf6773db1f32f0b178537ebbff7bdd2f1056c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 26 Aug 2021 11:53:02 +0200 Subject: [PATCH 45/46] RED-1920: Integrated entity-recognition-service --- .../client/EntityRecognitionClient.java | 18 ++++++ .../client/model/EntityRecogintionEntity.java | 19 +++++++ .../model/EntityRecognitionRequest.java | 18 ++++++ .../model/EntityRecognitionResponse.java | 21 +++++++ .../client/model/EntityRecognitionResult.java | 20 +++++++ .../model/EntityRecognitionSection.java | 16 ++++++ .../service/EntityRedactionService.java | 57 ++++++++++++++++--- .../settings/RedactionServiceSettings.java | 2 + .../src/test/resources/application.yml | 2 + 9 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResponse.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionSection.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java new file mode 100644 index 00000000..589e6dc3 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.server.client; + +import java.util.List; +import java.util.Map; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PostMapping; + +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecogintionEntity; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest; + +@FeignClient(name = "EntityRecognitionClient", url = "${entity-recognition-service.url}") +public interface EntityRecognitionClient { + + @PostMapping(value = "/find_authors", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + Map>> findAuthors(EntityRecognitionRequest entityRecognitionRequest); +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java new file mode 100644 index 00000000..b86a1b66 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecogintionEntity { + + private String value; + private int startOffset; + private int endOffset; + private String type; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java new file mode 100644 index 00000000..7e15bc69 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecognitionRequest { + + private List data; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResponse.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResponse.java new file mode 100644 index 00000000..af0d6beb --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResponse.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecognitionResponse { + + @Builder.Default + private Map> result = new HashMap<>(); + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java new file mode 100644 index 00000000..cb894b5d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecognitionResult { + + @Builder.Default + private Map> entities = new HashMap<>(); +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionSection.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionSection.java new file mode 100644 index 00000000..3172aa52 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionSection.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.redaction.v1.server.client.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class EntityRecognitionSection { + + private int sectionNumber; + private String text; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index c1dc3f11..67e86ca0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -2,14 +2,22 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.*; +import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecogintionEntity; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionResponse; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; + import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.kie.api.runtime.KieContainer; @@ -28,6 +36,8 @@ public class EntityRedactionService { private final DictionaryService dictionaryService; private final DroolsExecutionService droolsExecutionService; private final SurroundingWordsService surroundingWordsService; + private final EntityRecognitionClient entityRecognitionClient; + private final RedactionServiceSettings redactionServiceSettings; public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, @@ -58,7 +68,8 @@ public class EntityRedactionService { } - public Map> convertToEnititesPerPage(Set entities){ + public Map> convertToEnititesPerPage(Set entities) { + Map> entitiesPerPage = new HashMap<>(); for (Entity entity : entities) { Map> sequenceOnPage = new HashMap<>(); @@ -68,8 +79,7 @@ public class EntityRedactionService { } for (Map.Entry> entry : sequenceOnPage.entrySet()) { - entitiesPerPage - .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) + entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity @@ -80,17 +90,18 @@ public class EntityRedactionService { } - public Map> getHintsPerSection(Set entities, Dictionary dictionary){ + public Map> getHintsPerSection(Set entities, Dictionary dictionary) { + Map> hintsPerSectionNumber = new HashMap<>(); entities.stream().forEach(entity -> { if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) { - hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()) - .add(entity); + hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity); } }); return hintsPerSectionNumber; } + private Set findEntities(Document classifiedDoc, KieContainer kieContainer, ManualRedactions manualRedactions, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber, @@ -148,7 +159,8 @@ public class EntityRedactionService { } - public void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary){ + public void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) { + analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> { if (dictionary.isRecommendation(key)) { analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> { @@ -393,11 +405,16 @@ public class EntityRedactionService { } } + if (redactionServiceSettings.isEnableEntityRecognition() && !local) { + found.addAll(getAiEntities(sectionNumber, searchableString, headline)); + } + return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); } - private Image convertAndRecategorize(PdfImage pdfImage, int sectionNumber, String headline, ManualRedactions manualRedactions) { + private Image convertAndRecategorize(PdfImage pdfImage, int sectionNumber, String headline, + ManualRedactions manualRedactions) { Image image = Image.builder() .type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType() @@ -423,4 +440,28 @@ public class EntityRedactionService { return image; } + + private Set getAiEntities(int sectionNumber, String searchableString, String headline) { + + Set found = new HashSet<>(); + + Map>> response = entityRecognitionClient.findAuthors(EntityRecognitionRequest + .builder() + .data(List.of(EntityRecognitionSection.builder() + .sectionNumber(sectionNumber) + .text(searchableString) + .build())) + .build()); + + EntityRecognitionResponse entityRecognitionResponse = new EntityRecognitionResponse(response.get("result:")); + + if (entityRecognitionResponse.getResult() != null && entityRecognitionResponse.getResult() + .containsKey(String.valueOf(sectionNumber))) { + entityRecognitionResponse.getResult().get(String.valueOf(sectionNumber)).forEach(res -> { + found.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false)); + }); + } + return found; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java index dfc34079..901d8042 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java @@ -17,4 +17,6 @@ public class RedactionServiceSettings { private int analysisVersion = 1; + private boolean enableEntityRecognition = true; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml index 4b511179..72e05696 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml @@ -1,6 +1,7 @@ configuration-service.url: "http://configuration-service-v1:8080" image-service.url: "http://image-service-v1:8080" file-management-service.url: "http://file-management-service-v1:8080" +entity-recognition-service.url: "http://entity-recognition-service-v1:8080" ribbon: ConnectTimeout: 600000 @@ -17,3 +18,4 @@ platform.multi-tenancy: redaction-service: enable-image-classification: false + enable-entity-recognition: false From 0ab0ff5f501baa6fb4b3038b2aa4417a6729ce32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 26 Aug 2021 14:40:26 +0200 Subject: [PATCH 46/46] Fixed missing endpoint variable --- .../src/main/resources/application.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml index 671d3b20..d8aa5ace 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml @@ -4,6 +4,7 @@ info: configuration-service.url: "http://configuration-service-v1:8080" file-management-service.url: "http://file-management-service-v1:8080" image-service.url: "http://image-service-v1:8080" +entity-recognition-service.url: "http://entity-recognition-service-v1:8080" server: port: 8080