From 8b898a903853d7c8c80c290b7650c7dcf04c3794 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 14 May 2021 14:45:08 +0300 Subject: [PATCH 1/3] legal basis is now stored in redaction log --- redaction-service-v1/redaction-service-api-v1/pom.xml | 5 +++++ .../red/service/redaction/v1/model/RedactionLog.java | 7 ++++++- .../red/service/redaction/v1/model/RedactionLogEntry.java | 2 ++ .../redaction/v1/server/client/LegalBasisClient.java | 8 ++++++++ .../v1/server/redaction/service/ReanalyzeService.java | 6 ++++-- 5 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/LegalBasisClient.java diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml index a479f545..42f3ed40 100644 --- a/redaction-service-v1/redaction-service-api-v1/pom.xml +++ b/redaction-service-v1/redaction-service-api-v1/pom.xml @@ -17,5 +17,10 @@ spring-web true + + com.iqser.red.service + configuration-service-api-v1 + 2.5.6 + diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index b6c13b93..d8f3bfd6 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.model; +import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping; import lombok.Data; import lombok.NoArgsConstructor; @@ -11,6 +12,7 @@ public class RedactionLog { private List redactionLogEntry; + private List legalBasis; private long dictionaryVersion = -1; private long rulesVersion = -1; @@ -19,9 +21,12 @@ public class RedactionLog { private long dossierDictionaryVersion = -1; - public RedactionLog(List redactionLogEntry, long dictionaryVersion, long rulesVersion, String ruleSetId, long dossierDictionaryVersion) { + public RedactionLog(List redactionLogEntry, + List legalBasis, + long dictionaryVersion, long rulesVersion, String ruleSetId, long dossierDictionaryVersion) { this.redactionLogEntry = redactionLogEntry; + this.legalBasis = legalBasis; this.dictionaryVersion = dictionaryVersion; this.rulesVersion = rulesVersion; this.ruleSetId = ruleSetId; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index 6421c19e..deb34209 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.model; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import java.util.ArrayList; @@ -12,6 +13,7 @@ import java.util.List; @Builder @NoArgsConstructor @AllArgsConstructor +@EqualsAndHashCode(of = "id") public class RedactionLogEntry { private String id; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/LegalBasisClient.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/LegalBasisClient.java new file mode 100644 index 00000000..1a68ab35 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/LegalBasisClient.java @@ -0,0 +1,8 @@ +package com.iqser.red.service.redaction.v1.server.client; + +import com.iqser.red.service.configuration.v1.api.resource.LegalBasisMappingResource; +import org.springframework.cloud.openfeign.FeignClient; + +@FeignClient(name = "LegalBasisMappingResource", url = "${configuration-service.url}") +public interface LegalBasisClient extends LegalBasisMappingResource { +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 4c8d2e0a..12c10b7e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -5,6 +5,7 @@ import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.*; @@ -38,7 +39,7 @@ public class ReanalyzeService { private final PdfSegmentationService pdfSegmentationService; private final RedactionChangeLogService redactionChangeLogService; private final AnalyzeResponseService analyzeResponseService; - + private final LegalBasisClient legalBasisClient; public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { @@ -64,7 +65,8 @@ public class ReanalyzeService { log.info("Redaction analysis successful..."); - var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion() + var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getRuleSetId()); + var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis, classifiedDoc.getDictionaryVersion() .getRulesetVersion(), classifiedDoc.getRulesVersion(), analyzeRequest.getRuleSetId(), classifiedDoc.getDictionaryVersion() .getDossierVersion()); From 6a660c6a73d0b60f4c838cfba975378f515deffc Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 14 May 2021 14:50:06 +0300 Subject: [PATCH 2/3] adjusted tests --- .../service/redaction/v1/server/RedactionIntegrationTest.java | 4 ++++ .../server/redaction/service/EntityRedactionServiceTest.java | 4 ++++ .../v1/server/segmentation/PdfSegmentationServiceTest.java | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 8b21791f..6f14d819 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -9,6 +9,7 @@ import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.memory.MemoryStats; @@ -114,6 +115,9 @@ public class RedactionIntegrationTest { @MockBean private RabbitTemplate rabbitTemplate; + @MockBean + private LegalBasisClient legalBasisClient; + private final Map> dictionary = new HashMap<>(); private final Map> dossierDictionary = new HashMap<>(); private final Map typeColorMap = new HashMap<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java index b6cb00ca..512710d0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java @@ -7,6 +7,7 @@ import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; @@ -78,6 +79,9 @@ public class EntityRedactionServiceTest { @MockBean private AmazonS3 amazonS3; + @MockBean + private LegalBasisClient legalBasisClient; + private final static String TEST_RULESET_ID = "123"; @Configuration diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java index 44842b7d..2a2c3a37 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java @@ -5,6 +5,7 @@ import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; @@ -62,6 +63,9 @@ public class PdfSegmentationServiceTest { @MockBean private RabbitTemplate rabbitTemplate; + @MockBean + private LegalBasisClient legalBasisClient; + @Configuration @EnableAutoConfiguration(exclude = { RabbitAutoConfiguration.class}) public static class TestConfiguration { From 513501abf476dc7f08446bb3035da888d2fd30a2 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 14 May 2021 15:00:22 +0300 Subject: [PATCH 3/3] save legal basis version on redaction log --- .../redaction-service-api-v1/pom.xml | 8 +++++++- .../redaction/v1/model/RedactionLog.java | 20 +++++-------------- .../redaction-service-server-v1/pom.xml | 9 ++++----- .../redaction/service/ReanalyzeService.java | 9 ++++++--- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml index 42f3ed40..cb0d850a 100644 --- a/redaction-service-v1/redaction-service-api-v1/pom.xml +++ b/redaction-service-v1/redaction-service-api-v1/pom.xml @@ -20,7 +20,13 @@ com.iqser.red.service configuration-service-api-v1 - 2.5.6 + 2.7.0 + + + com.iqser.red.service + file-management-service-api-v1 + + diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index d8f3bfd6..9c28f5fc 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -1,37 +1,27 @@ package com.iqser.red.service.redaction.v1.model; import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping; +import lombok.AllArgsConstructor; import lombok.Data; -import lombok.NoArgsConstructor; import java.util.List; @Data -@NoArgsConstructor +@AllArgsConstructor public class RedactionLog { private List redactionLogEntry; - private List legalBasis; - private long dictionaryVersion = -1; - private long rulesVersion = -1; private String ruleSetId; + private long dictionaryVersion = -1; + private long rulesVersion = -1; private long dossierDictionaryVersion = -1; + private long legalBasisVersion = -1; - public RedactionLog(List redactionLogEntry, - List legalBasis, - long dictionaryVersion, long rulesVersion, String ruleSetId, long dossierDictionaryVersion) { - this.redactionLogEntry = redactionLogEntry; - this.legalBasis = legalBasis; - this.dictionaryVersion = dictionaryVersion; - this.rulesVersion = rulesVersion; - this.ruleSetId = ruleSetId; - this.dossierDictionaryVersion = dossierDictionaryVersion; - } } diff --git a/redaction-service-v1/redaction-service-server-v1/pom.xml b/redaction-service-v1/redaction-service-server-v1/pom.xml index 1975a136..ddb9b436 100644 --- a/redaction-service-v1/redaction-service-server-v1/pom.xml +++ b/redaction-service-v1/redaction-service-server-v1/pom.xml @@ -21,11 +21,6 @@ redaction-service-api-v1 ${project.version} - - com.iqser.red.service - configuration-service-api-v1 - 2.5.6 - com.iqser.red.service file-management-service-api-v1 @@ -35,6 +30,10 @@ com.iqser.red.service redaction-service-api-v1 + + com.iqser.red.service + configuration-service-api-v1 + diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 12c10b7e..511d689e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -66,9 +66,12 @@ public class ReanalyzeService { log.info("Redaction analysis successful..."); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getRuleSetId()); - var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis, classifiedDoc.getDictionaryVersion() - .getRulesetVersion(), classifiedDoc.getRulesVersion(), analyzeRequest.getRuleSetId(), classifiedDoc.getDictionaryVersion() - .getDossierVersion()); + var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis, + analyzeRequest.getRuleSetId(), + classifiedDoc.getDictionaryVersion().getRulesetVersion(), + classifiedDoc.getRulesVersion(), + classifiedDoc.getDictionaryVersion().getDossierVersion(), + legalBasisClient.getVersion(analyzeRequest.getRuleSetId())); log.info("Analyzed with rules {} and dictionary {} for ruleSet: {}", classifiedDoc.getRulesVersion(), classifiedDoc .getDictionaryVersion(), analyzeRequest.getRuleSetId());