From e01074ef27b4aaabd15cd33a3cc2f29bc95066de Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 31 May 2021 10:56:48 +0300 Subject: [PATCH] migrated ruleset and projectid to new naming --- .../redaction/v1/model/AnalyzeRequest.java | 4 +- .../redaction/v1/model/AnalyzeResult.java | 2 +- .../redaction/v1/model/AnnotateRequest.java | 2 +- .../v1/model/RedactionChangeLog.java | 2 +- .../redaction/v1/model/RedactionLog.java | 2 +- .../redaction/v1/model/RedactionRequest.java | 4 +- .../v1/resources/RedactionResource.java | 4 +- .../controller/RedactionController.java | 22 +-- .../queue/RedactionMessageReceiver.java | 4 +- .../model/DictionaryRepresentation.java | 2 +- .../redaction/model/DictionaryVersion.java | 2 +- .../service/AnalyzeResponseService.java | 4 +- .../redaction/service/DictionaryService.java | 87 ++++++------ .../service/DroolsExecutionService.java | 38 +++--- .../service/EntityRedactionService.java | 12 +- .../redaction/service/ReanalyzeService.java | 52 +++---- .../service/RedactionChangeLogService.java | 8 +- .../service/RedactionLogCreatorService.java | 90 ++++++------ .../storage/RedactionStorageService.java | 20 +-- .../v1/server/RedactionIntegrationTest.java | 116 ++++++++-------- .../service/EntityRedactionServiceTest.java | 128 +++++++++--------- 21 files changed, 300 insertions(+), 305 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 113bdd43..4aa290e9 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -13,9 +13,9 @@ import java.time.OffsetDateTime; @AllArgsConstructor public class AnalyzeRequest { - private String projectId; + private String dossierId; private String fileId; - private String ruleSetId; + private String dossierTemplateId; private boolean reanalyseOnlyIfPossible; private ManualRedactions manualRedactions; private OffsetDateTime lastProcessed; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java index 2f1ffbee..1ce9d759 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java @@ -11,7 +11,7 @@ import lombok.NoArgsConstructor; @AllArgsConstructor public class AnalyzeResult { - private String projectId; + private String dossierId; private String fileId; private long duration; private int numberOfPages; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnnotateRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnnotateRequest.java index 4f65d74e..30d0a62f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnnotateRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnnotateRequest.java @@ -11,6 +11,6 @@ import lombok.NoArgsConstructor; @AllArgsConstructor public class AnnotateRequest { - private String projectId; + private String dossierId; private String fileId; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java index 1270b800..10a41b05 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java @@ -17,6 +17,6 @@ public class RedactionChangeLog { private long dictionaryVersion = -1; private long rulesVersion = -1; - private String ruleSetId; + private String dossierTemplateId; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index 9c28f5fc..7c916084 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -13,7 +13,7 @@ public class RedactionLog { private List redactionLogEntry; private List legalBasis; - private String ruleSetId; + private String dossierTemplateId; private long dictionaryVersion = -1; private long rulesVersion = -1; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java index fd525887..263e7692 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java @@ -11,8 +11,8 @@ import lombok.NoArgsConstructor; @AllArgsConstructor public class RedactionRequest { - private String projectId; + private String dossierId; private String fileId; - private String ruleSetId; + private String dossierTemplateId; private ManualRedactions manualRedactions; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index de766ba5..cdf45fcb 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -10,7 +10,7 @@ public interface RedactionResource { String SERVICE_NAME = "redaction-service-v1"; - String RULE_SET_PARAMETER_NAME = "ruleSetId"; + String RULE_SET_PARAMETER_NAME = "dossierTemplateId"; String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}"; @@ -27,7 +27,7 @@ public interface RedactionResource { RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest); @PostMapping(value = "/rules/update" + RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE) - void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId); + void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId); @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE) void testRules(@RequestBody String rules); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 31f8583b..12381286 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -43,14 +43,14 @@ public class RedactionController implements RedactionResource { public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { - var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getProjectId(), annotateRequest.getFileId(), FileType.ORIGIN)); - var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getProjectId(), annotateRequest.getFileId()); - var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getProjectId(), annotateRequest.getFileId()); + var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN)); + var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId()); + var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId()); try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) { pdDocument.setAllSecurityToBeRemoved(true); - dictionaryService.updateDictionary(redactionLog.getRuleSetId(), annotateRequest.getProjectId()); + dictionaryService.updateDictionary(redactionLog.getDossierTemplateId(), annotateRequest.getDossierId()); annotationService.annotate(pdDocument, redactionLog, sectionsGrid); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { @@ -66,11 +66,11 @@ public class RedactionController implements RedactionResource { @Override public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) { - var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); + var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN)); try { Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream); - storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); + storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN)); try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) { pdDocument.setAllSecurityToBeRemoved(true); @@ -91,11 +91,11 @@ public class RedactionController implements RedactionResource { @Override public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) { - var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); + var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN)); try { Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream); - storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); + storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN)); try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) { pdDocument.setAllSecurityToBeRemoved(true); @@ -120,7 +120,7 @@ public class RedactionController implements RedactionResource { Document classifiedDoc; try { - var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); + var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN)); classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, true); } catch (Exception e) { throw new RedactionException(e); @@ -143,9 +143,9 @@ public class RedactionController implements RedactionResource { @Override - public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId) { + public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId) { - droolsExecutionService.updateRules(ruleSetId); + droolsExecutionService.updateRules(dossierTemplateId); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index 2a08e4c8..667e21fc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -38,7 +38,7 @@ public class RedactionMessageReceiver { } log.info("Successfully analyzed {}", analyzeRequest); - fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), result); + fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result); } @RabbitHandler @@ -48,7 +48,7 @@ public class RedactionMessageReceiver { var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class); log.info("Failed to process analyze request: {}", analyzeRequest); - fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getProjectId(), analyzeRequest.getFileId()); + fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java index 0f7b6820..1f1e6da7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java @@ -10,7 +10,7 @@ import java.util.Map; @Data public class DictionaryRepresentation { - private String ruleSetId; + private String dossierTemplateId; private long dictionaryVersion = -1; private List dictionary = new ArrayList<>(); private float[] defaultColor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java index 6a69bb60..067306a7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java @@ -11,6 +11,6 @@ import lombok.NoArgsConstructor; @AllArgsConstructor public class DictionaryVersion { - long rulesetVersion; + long dossierTemplateVersion; long dossierVersion; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index c6572912..bae6d1d0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -9,7 +9,7 @@ import org.springframework.stereotype.Service; @Service public class AnalyzeResponseService { - public AnalyzeResult createAnalyzeResponse(String projectId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { + public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { boolean hasHints = redactionLog.getRedactionLogEntry().stream().anyMatch(RedactionLogEntry::isHint); boolean hasRequests = redactionLog.getRedactionLogEntry() @@ -31,7 +31,7 @@ public class AnalyzeResponseService { .isEmpty() && redactionChangeLog.getRedactionLogEntry().stream().anyMatch(entry -> !entry.getType().equals("false_positive")); return AnalyzeResult.builder() - .projectId(projectId) + .dossierId(dossierId) .fileId(fileId) .duration(duration) .numberOfPages(pageCount) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index d9577bd0..bca42897 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -1,19 +1,12 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import static com.iqser.red.service.configuration.v1.api.resource.DictionaryResource.GLOBAL_DOSSIER; - import com.iqser.red.service.configuration.v1.api.model.Colors; import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry; import com.iqser.red.service.configuration.v1.api.model.TypeResponse; import com.iqser.red.service.configuration.v1.api.model.TypeResult; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; - +import com.iqser.red.service.redaction.v1.server.redaction.model.*; import feign.FeignException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -25,6 +18,8 @@ import java.awt.Color; import java.util.*; import java.util.stream.Collectors; +import static com.iqser.red.service.configuration.v1.api.resource.DictionaryResource.GLOBAL_DOSSIER; + @Slf4j @Service @RequiredArgsConstructor @@ -36,30 +31,30 @@ public class DictionaryService { private final Map dictionariesByDossier = new HashMap<>(); - public DictionaryVersion updateDictionary(String ruleSetId, String dossierId) { + public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) { - long rulesetDictionaryVersion = dictionaryClient.getVersion(ruleSetId, GLOBAL_DOSSIER); - var rulesetDictionary = dictionariesByRuleSets.get(ruleSetId); - if (rulesetDictionary == null || rulesetDictionaryVersion > rulesetDictionary.getDictionaryVersion()) { - updateDictionaryEntry(ruleSetId, rulesetDictionaryVersion, GLOBAL_DOSSIER); + long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER); + var dossierTemplateDictionary = dictionariesByRuleSets.get(dossierTemplateId); + if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) { + updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, GLOBAL_DOSSIER); } - long dossierDictionaryVersion = dictionaryClient.getVersion(ruleSetId, dossierId); + long dossierDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, dossierId); var dossierDictionary = dictionariesByDossier.get(dossierId); if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) { - updateDictionaryEntry(ruleSetId, dossierDictionaryVersion, dossierId); + updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, dossierId); } - return DictionaryVersion.builder().rulesetVersion(rulesetDictionaryVersion).dossierVersion(dossierDictionaryVersion).build(); + return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build(); } - public DictionaryIncrement getDictionaryIncrements(String ruleSetId, DictionaryVersion fromVersion, String dossierId) { + public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) { - DictionaryVersion version = updateDictionary(ruleSetId, dossierId); + DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId); Set newValues = new HashSet<>(); - List dictionaryModels = dictionariesByRuleSets.get(ruleSetId).getDictionary(); + List dictionaryModels = dictionariesByRuleSets.get(dossierTemplateId).getDictionary(); dictionaryModels.forEach(dictionaryModel -> { dictionaryModel.getEntries().forEach(dictionaryEntry -> { if (dictionaryEntry.getVersion() > fromVersion.getRulesetVersion()) { @@ -83,12 +78,12 @@ public class DictionaryService { } - private void updateDictionaryEntry(String ruleSetId, long version, String dossierId) { + private void updateDictionaryEntry(String dossierTemplateId, long version, String dossierId) { try { DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation(); - TypeResponse typeResponse = dictionaryClient.getAllTypes(ruleSetId, dossierId); + TypeResponse typeResponse = dictionaryClient.getAllTypes(dossierTemplateId, dossierId); if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) { List dictionary = typeResponse.getTypes() @@ -100,18 +95,18 @@ public class DictionaryService { dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm)); - Colors colors = dictionaryClient.getColors(ruleSetId); + Colors colors = dictionaryClient.getColors(dossierTemplateId); dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor())); dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd())); dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove())); dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted())); - dictionaryRepresentation.setRuleSetId(ruleSetId); + dictionaryRepresentation.setDossierTemplateId(dossierTemplateId); dictionaryRepresentation.setDictionaryVersion(version); dictionaryRepresentation.setDictionary(dictionary); if(dossierId.equals(GLOBAL_DOSSIER)) { - dictionariesByRuleSets.put(ruleSetId, dictionaryRepresentation); + dictionariesByRuleSets.put(dossierTemplateId, dictionaryRepresentation); } else { dictionariesByDossier.put(dossierId, dictionaryRepresentation); } @@ -123,12 +118,12 @@ public class DictionaryService { } - public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) { + public void updateExternalDictionary(Dictionary dictionary, String dossierTemplateId) { dictionary.getDictionaryModels().forEach(dm -> { if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) { - dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false, GLOBAL_DOSSIER); - long externalVersion = dictionaryClient.getVersion(ruleSetId, GLOBAL_DOSSIER); + dictionaryClient.addEntries(dm.getType(), dossierTemplateId, new ArrayList<>(dm.getLocalEntries()), false, GLOBAL_DOSSIER); + long externalVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER); if (externalVersion == dictionary.getVersion().getRulesetVersion() + 1) { dictionary.getVersion().setRulesetVersion(externalVersion); } @@ -156,9 +151,9 @@ public class DictionaryService { } - public boolean isCaseInsensitiveDictionary(String type, String ruleSetId) { + public boolean isCaseInsensitiveDictionary(String type, String dossierTemplateId) { - DictionaryModel dictionaryModel = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type); + DictionaryModel dictionaryModel = dictionariesByRuleSets.get(dossierTemplateId).getLocalAccessMap().get(type); if (dictionaryModel != null) { return dictionaryModel.isCaseInsensitive(); } @@ -166,19 +161,19 @@ public class DictionaryService { } - public float[] getColor(String type, String ruleSetId) { + public float[] getColor(String type, String dossierTemplateId) { - DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type); + DictionaryModel model = dictionariesByRuleSets.get(dossierTemplateId).getLocalAccessMap().get(type); if (model != null) { return model.getColor(); } - return dictionariesByRuleSets.get(ruleSetId).getDefaultColor(); + return dictionariesByRuleSets.get(dossierTemplateId).getDefaultColor(); } - public boolean isHint(String type, String ruleSetId) { + public boolean isHint(String type, String dossierTemplateId) { - DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type); + DictionaryModel model = dictionariesByRuleSets.get(dossierTemplateId).getLocalAccessMap().get(type); if (model != null) { return model.isHint(); } @@ -186,9 +181,9 @@ public class DictionaryService { } - public boolean isRecommendation(String type, String ruleSetId) { + public boolean isRecommendation(String type, String dossierTemplateId) { - DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type); + DictionaryModel model = dictionariesByRuleSets.get(dossierTemplateId).getLocalAccessMap().get(type); if (model != null) { return model.isRecommendation(); } @@ -196,12 +191,12 @@ public class DictionaryService { } - public Dictionary getDeepCopyDictionary(String ruleSetId, String dossierId) { + public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) { List copy = new ArrayList<>(); - var rulesetRepresentation = dictionariesByRuleSets.get(ruleSetId); - rulesetRepresentation.getDictionary().forEach(dm -> { + var dossierTemplateRepresentation = dictionariesByRuleSets.get(dossierTemplateId); + dossierTemplateRepresentation.getDictionary().forEach(dm -> { copy.add(SerializationUtils.clone(dm)); }); @@ -215,25 +210,25 @@ public class DictionaryService { dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); } - return new Dictionary(copy, DictionaryVersion.builder().rulesetVersion(rulesetRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); + return new Dictionary(copy, DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); } - public float[] getRequestRemoveColor(String ruleSetId) { + public float[] getRequestRemoveColor(String dossierTemplateId) { - return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor(); + return dictionariesByRuleSets.get(dossierTemplateId).getRequestAddColor(); } - public float[] getNotRedactedColor(String ruleSetId) { + public float[] getNotRedactedColor(String dossierTemplateId) { - return dictionariesByRuleSets.get(ruleSetId).getNotRedactedColor(); + return dictionariesByRuleSets.get(dossierTemplateId).getNotRedactedColor(); } - public float[] getRequestAddColor(String ruleSetId) { + public float[] getRequestAddColor(String dossierTemplateId) { - return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor(); + return dictionariesByRuleSets.get(dossierTemplateId).getRequestAddColor(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 708efa12..bd26cc12 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -28,14 +28,14 @@ public class DroolsExecutionService { private final Map kieContainers = new HashMap<>(); - private final Map rulesVersionPerRuleSetId = new HashMap<>(); + private final Map rulesVersionPerDossierTemplateId = new HashMap<>(); - public KieContainer getKieContainer(String ruleSetId) { + public KieContainer getKieContainer(String dossierTemplateId) { - KieContainer container = kieContainers.get(ruleSetId); + KieContainer container = kieContainers.get(dossierTemplateId); if (container == null) { - return createOrUpdateKieContainer(ruleSetId); + return createOrUpdateKieContainer(dossierTemplateId); } else { return container; } @@ -55,43 +55,43 @@ public class DroolsExecutionService { } - public KieContainer updateRules(String ruleSetId) { + public KieContainer updateRules(String dossierTemplateId) { - long version = rulesClient.getVersion(ruleSetId); - Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId); + long version = rulesClient.getVersion(dossierTemplateId); + Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId); if (rulesVersion == null) { rulesVersion = -1L; } if (version > rulesVersion.longValue()) { - rulesVersionPerRuleSetId.put(ruleSetId, version); - return createOrUpdateKieContainer(ruleSetId); + rulesVersionPerDossierTemplateId.put(dossierTemplateId, version); + return createOrUpdateKieContainer(dossierTemplateId); } - return getKieContainer(ruleSetId); + return getKieContainer(dossierTemplateId); } - private KieContainer createOrUpdateKieContainer(String ruleSetId) { + private KieContainer createOrUpdateKieContainer(String dossierTemplateId) { try { - RulesResponse rules = rulesClient.getRules(ruleSetId); + RulesResponse rules = rulesClient.getRules(dossierTemplateId); if (rules == null || StringUtils.isEmpty(rules.getRules())) { throw new RuntimeException("Rules cannot be empty."); } KieServices kieServices = KieServices.Factory.get(); - KieModule kieModule = getKieModule(ruleSetId, rules.getRules(), kieServices); + KieModule kieModule = getKieModule(dossierTemplateId, rules.getRules(), kieServices); - var container = kieContainers.get(ruleSetId); + var container = kieContainers.get(dossierTemplateId); if (container != null) { container.updateToVersion(kieModule.getReleaseId()); return container; } container = kieServices.newKieContainer(kieModule.getReleaseId()); - kieContainers.put(ruleSetId, container); + kieContainers.put(dossierTemplateId, container); return container; } catch (Exception e) { throw new RulesValidationException("Could not update rules: " + e.getMessage(), e); @@ -100,11 +100,11 @@ public class DroolsExecutionService { } - private KieModule getKieModule(String ruleSetId, String rules, KieServices kieServices) { + private KieModule getKieModule(String dossierTemplateId, String rules, KieServices kieServices) { KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8)); - kieFileSystem.write("src/main/resources/drools/rules" + ruleSetId + ".drl", kieServices.getResources() + kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources() .newInputStreamResource(input)); KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); kieBuilder.buildAll(); @@ -122,9 +122,9 @@ public class DroolsExecutionService { } - public long getRulesVersion(String ruleSetId) { + public long getRulesVersion(String dossierTemplateId) { - Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId); + Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId); if (rulesVersion == null) { return -1; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 3e8875af..02da3aa7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -33,13 +33,13 @@ public class EntityRedactionService { private final SurroundingWordsService surroundingWordsService; - public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions, String dossierId) { + public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId) { - dictionaryService.updateDictionary(ruleSetId, dossierId); - KieContainer container = droolsExecutionService.updateRules(ruleSetId); - long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId); + dictionaryService.updateDictionary(dossierTemplateId, dossierId); + KieContainer container = droolsExecutionService.updateRules(dossierTemplateId); + long rulesVersion = droolsExecutionService.getRulesVersion(dossierTemplateId); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId, dossierId); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(dossierTemplateId, dossierId); Set documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null)); @@ -75,7 +75,7 @@ public class EntityRedactionService { } } - dictionaryService.updateExternalDictionary(dictionary, ruleSetId); + dictionaryService.updateExternalDictionary(dictionary, dossierTemplateId); classifiedDoc.setDictionaryVersion(dictionary.getVersion()); classifiedDoc.setRulesVersion(rulesVersion); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 511d689e..9c0d75b8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -50,7 +50,7 @@ public class ReanalyzeService { try { var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest - .getProjectId(), analyzeRequest.getFileId(), FileType.ORIGIN)); + .getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN)); classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream); pageCount = classifiedDoc.getPages().size(); } catch (Exception e) { @@ -58,35 +58,35 @@ public class ReanalyzeService { } log.info("Document structure analysis successful, starting redaction analysis..."); - entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getRuleSetId(), analyzeRequest.getManualRedactions(), analyzeRequest - .getProjectId()); + entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest + .getDossierId()); redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest - .getRuleSetId()); + .getDossierTemplateId()); log.info("Redaction analysis successful..."); - var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getRuleSetId()); + var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis, - analyzeRequest.getRuleSetId(), + analyzeRequest.getDossierTemplateId(), classifiedDoc.getDictionaryVersion().getRulesetVersion(), classifiedDoc.getRulesVersion(), classifiedDoc.getDictionaryVersion().getDossierVersion(), - legalBasisClient.getVersion(analyzeRequest.getRuleSetId())); + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); - log.info("Analyzed with rules {} and dictionary {} for ruleSet: {}", classifiedDoc.getRulesVersion(), classifiedDoc - .getDictionaryVersion(), analyzeRequest.getRuleSetId()); + log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc + .getDictionaryVersion(), analyzeRequest.getDossierTemplateId()); // first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data - var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), redactionLog); + var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); // store redactionLog - redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); - redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc .getSectionText())); - redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc .getSectionGrid()); long duration = System.currentTimeMillis() - startTime; - return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog); + return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog); } @@ -95,16 +95,16 @@ public class ReanalyzeService { long startTime = System.currentTimeMillis(); - var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId()); - var text = redactionStorageService.getText(analyzeRequest.getProjectId(), analyzeRequest.getFileId()); + var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); // not yet ready for reanalysis if (redactionLog == null || text == null || text.getNumberOfPages() == 0) { return analyze(analyzeRequest); } - DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getRuleSetId(), new DictionaryVersion(redactionLog - .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getProjectId()); + DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog + .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); Set manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions()); Map> comments = null; @@ -164,9 +164,9 @@ public class ReanalyzeService { //-- - KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getRuleSetId()); + KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getRuleSetId(), analyzeRequest.getProjectId()); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); List sectionSearchableTextPairs = new ArrayList<>(); for (SectionText reanalysisSection : reanalysisSections) { @@ -229,16 +229,16 @@ public class ReanalyzeService { for (int page = 1; page <= text.getNumberOfPages(); page++) { if (entitiesPerPage.get(page) != null) { newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest - .getRuleSetId())); + .getDossierTemplateId())); } if (imagesPerPage.get(page) != null) { newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest - .getRuleSetId())); + .getDossierTemplateId())); } newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest - .getRuleSetId())); + .getDossierTemplateId())); } redactionLog.getRedactionLogEntry() @@ -256,12 +256,12 @@ public class ReanalyzeService { redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getRulesetVersion()); redactionLog.setDossierDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierVersion()); - var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), redactionLog); - redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); + var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); long duration = System.currentTimeMillis() - startTime; - return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), duration, text + return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text .getNumberOfPages(), redactionLog, changeLog); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index 72bb7954..1d586ad1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -22,12 +22,12 @@ public class RedactionChangeLogService { private final RedactionStorageService redactionStorageService; - public RedactionChangeLog createAndStoreChangeLog(String projectId, String fileId, RedactionLog currentRedactionLog) { + public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) { try { - RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(projectId, fileId); + RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId); var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog); - redactionStorageService.storeObject(projectId, fileId, FileType.REDACTION_CHANGELOG, changeLog); + redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog); return changeLog; } catch (Exception e) { log.debug("Previous redaction log not available"); @@ -58,7 +58,7 @@ public class RedactionChangeLogService { .collect(Collectors.toList())); return new RedactionChangeLog(changeLogEntries, currentRedactionLog.getDictionaryVersion(), currentRedactionLog.getRulesVersion(), currentRedactionLog - .getRuleSetId()); + .getDossierTemplateId()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 4a591420..9aef4dc7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -32,7 +32,7 @@ public class RedactionLogCreatorService { public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions, - String ruleSetId) { + String dossierTemplateId) { Set manualRedactionPages = getManualRedactionPages(manualRedactions); @@ -42,24 +42,24 @@ public class RedactionLogCreatorService { if (classifiedDoc.getEntities().get(page) != null) { classifiedDoc.getRedactionLogEntities() - .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, ruleSetId)); + .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId)); } if (manualRedactionPages.contains(page)) { classifiedDoc.getRedactionLogEntities() - .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, ruleSetId)); + .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId)); } if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) { classifiedDoc.getRedactionLogEntities() - .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, ruleSetId)); + .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId)); } } } public List addImageEntries(Map> images, ManualRedactions manualRedactions, - int pageNumber, String ruleSetId) { + int pageNumber, String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -69,14 +69,14 @@ public class RedactionLogCreatorService { RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() .id(id) - .color(getColorForImage(image, ruleSetId, false)) + .color(getColorForImage(image, dossierTemplateId, false)) .isImage(true) .type(image.getType()) .redacted(image.isRedaction()) .reason(image.getRedactionReason()) .legalBasis(image.getLegalBasis()) .matchedRule(image.getMatchedRule()) - .isHint(dictionaryService.isHint(image.getType(), ruleSetId)) + .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId)) .manual(false) .isDictionaryEntry(false) .isRecommendation(false) @@ -96,11 +96,11 @@ public class RedactionLogCreatorService { redactionLogEntry.setRedacted(false); redactionLogEntry.setStatus(Status.APPROVED); manualOverrideReason = image.getRedactionReason() + ", removed by manual override"; - redactionLogEntry.setColor(getColorForImage(image, ruleSetId, false)); + redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false)); } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = image.getRedactionReason() + ", requested to remove"; redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(image, ruleSetId, true)); + redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true)); } else { redactionLogEntry.setStatus(Status.DECLINED); } @@ -121,13 +121,13 @@ public class RedactionLogCreatorService { image.setRedaction(true); redactionLogEntry.setRedacted(true); redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColorForImage(image, ruleSetId, false)); + redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false)); manualOverrideReason = image.getRedactionReason() + ", forced by manual override"; redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = image.getRedactionReason() + ", requested to force redact"; redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(image, ruleSetId, true)); + redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true)); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else { redactionLogEntry.setStatus(Status.DECLINED); @@ -166,7 +166,7 @@ public class RedactionLogCreatorService { public List addEntries(Map> entities, ManualRedactions manualRedactions, - int page, String ruleSetId) { + int page, String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -180,7 +180,7 @@ public class RedactionLogCreatorService { for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, ruleSetId); + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId); if (processedIds.contains(entityPositionSequence.getId())) { // TODO refactor this outer loop jump as soon as we have the time. @@ -199,11 +199,11 @@ public class RedactionLogCreatorService { redactionLogEntry.setRedacted(false); redactionLogEntry.setStatus(Status.APPROVED); manualOverrideReason = entity.getRedactionReason() + ", removed by manual override"; - redactionLogEntry.setColor(getColor(entity, ruleSetId, false)); + redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false)); } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = entity.getRedactionReason() + ", requested to remove"; redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(entity, ruleSetId, true)); + redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true)); } else { redactionLogEntry.setStatus(Status.DECLINED); } @@ -224,13 +224,13 @@ public class RedactionLogCreatorService { entity.setRedaction(true); redactionLogEntry.setRedacted(true); redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColor(entity, ruleSetId, false)); + redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false)); manualOverrideReason = entity.getRedactionReason() + ", forced by manual override"; redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = entity.getRedactionReason() + ", requested to force redact"; redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(entity, ruleSetId, true)); + redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true)); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else { redactionLogEntry.setStatus(Status.DECLINED); @@ -299,7 +299,7 @@ public class RedactionLogCreatorService { public List addManualAddEntries(Set manualAdds, Map> comments, int page, - String ruleSetId) { + String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -311,7 +311,7 @@ public class RedactionLogCreatorService { String id = manualRedactionEntry.getId(); - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, ruleSetId); + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId); List rectanglesOnPage = new ArrayList<>(); for (Rectangle rectangle : manualRedactionEntry.getPositions()) { @@ -338,11 +338,11 @@ public class RedactionLogCreatorService { private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, - String ruleSetId) { + String dossierTemplateId) { return RedactionLogEntry.builder() .id(id) - .color(getColorForManualAdd(manualRedactionEntry.getType(), ruleSetId, manualRedactionEntry.getStatus())) + .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) .reason(manualRedactionEntry.getReason()) .legalBasis(manualRedactionEntry.getLegalBasis()) .value(manualRedactionEntry.getValue()) @@ -360,17 +360,17 @@ public class RedactionLogCreatorService { } - private RedactionLogEntry createRedactionLogEntry(Entity entity, String ruleSetId) { + private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) { return RedactionLogEntry.builder() - .color(getColor(entity, ruleSetId, false)) + .color(getColor(entity, dossierTemplateId, false)) .reason(entity.getRedactionReason()) .legalBasis(entity.getLegalBasis()) .value(entity.getWord()) .type(entity.getType()) .redacted(entity.isRedaction()) - .isHint(isHint(entity, ruleSetId)) - .isRecommendation(isRecommendation(entity, ruleSetId)) + .isHint(isHint(entity, dossierTemplateId)) + .isRecommendation(isRecommendation(entity, dossierTemplateId)) .section(entity.getHeadline()) .sectionNumber(entity.getSectionNumber()) .matchedRule(entity.getMatchedRule()) @@ -384,56 +384,56 @@ public class RedactionLogCreatorService { } - private float[] getColor(Entity entity, String ruleSetId, boolean requestedToRemove) { + private float[] getColor(Entity entity, String dossierTemplateId, boolean requestedToRemove) { if (requestedToRemove) { - return dictionaryService.getRequestRemoveColor(ruleSetId); + return dictionaryService.getRequestRemoveColor(dossierTemplateId); } - if (!entity.isRedaction() && !isHint(entity, ruleSetId)) { - return dictionaryService.getNotRedactedColor(ruleSetId); + if (!entity.isRedaction() && !isHint(entity, dossierTemplateId)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); } - return dictionaryService.getColor(entity.getType(), ruleSetId); + return dictionaryService.getColor(entity.getType(), dossierTemplateId); } - private float[] getColorForManualAdd(String type, String ruleSetId, Status status) { + private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) { if (status.equals(Status.REQUESTED)) { - return dictionaryService.getRequestAddColor(ruleSetId); + return dictionaryService.getRequestAddColor(dossierTemplateId); } else if (status.equals(Status.DECLINED)) { - return dictionaryService.getNotRedactedColor(ruleSetId); + return dictionaryService.getNotRedactedColor(dossierTemplateId); } - return getColor(type, ruleSetId); + return getColor(type, dossierTemplateId); } - private float[] getColor(String type, String ruleSetId) { + private float[] getColor(String type, String dossierTemplateId) { - return dictionaryService.getColor(type, ruleSetId); + return dictionaryService.getColor(type, dossierTemplateId); } - private float[] getColorForImage(Image image, String ruleSetId, boolean requestedToRemove) { + private float[] getColorForImage(Image image, String dossierTemplateId, boolean requestedToRemove) { if (requestedToRemove) { - return dictionaryService.getRequestRemoveColor(ruleSetId); + return dictionaryService.getRequestRemoveColor(dossierTemplateId); } - if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), ruleSetId)) { - return dictionaryService.getNotRedactedColor(ruleSetId); + if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), dossierTemplateId)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); } - return dictionaryService.getColor(image.getType(), ruleSetId); + return dictionaryService.getColor(image.getType(), dossierTemplateId); } - private boolean isHint(Entity entity, String ruleSetId) { + private boolean isHint(Entity entity, String dossierTemplateId) { - return dictionaryService.isHint(entity.getType(), ruleSetId); + return dictionaryService.isHint(entity.getType(), dossierTemplateId); } - private boolean isRecommendation(Entity entity, String ruleSetId) { + private boolean isRecommendation(Entity entity, String dossierTemplateId) { - return dictionaryService.isRecommendation(entity.getType(), ruleSetId); + return dictionaryService.isRecommendation(entity.getType(), dossierTemplateId); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index f350b7ac..021beeff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -32,16 +32,16 @@ public class RedactionStorageService { @SneakyThrows - public void storeObject(String projectId, String fileId, FileType fileType, Object any) { - storageService.storeObject(StorageIdUtils.getStorageId(projectId, fileId, fileType), objectMapper.writeValueAsBytes(any)); + public void storeObject(String dossierId, String fileId, FileType fileType, Object any) { + storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any)); } - public RedactionLog getRedactionLog(String projectId, String fileId) { + public RedactionLog getRedactionLog(String dossierId, String fileId) { InputStreamResource inputStreamResource; try { - inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(projectId, fileId, FileType.REDACTION_LOG)); + inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG)); } catch (StorageObjectDoesNotExist e) { log.debug("Text not available."); return null; @@ -55,11 +55,11 @@ public class RedactionStorageService { } - public Text getText(String projectId, String fileId) { + public Text getText(String dossierId, String fileId) { InputStreamResource inputStreamResource; try { - inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(projectId, fileId, FileType.TEXT)); + inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT)); } catch (StorageObjectDoesNotExist e) { log.debug("Text not available."); return null; @@ -73,9 +73,9 @@ public class RedactionStorageService { } - public SectionGrid getSectionGrid(String projectId, String fileId) { + public SectionGrid getSectionGrid(String dossierId, String fileId) { - var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(projectId, fileId, FileType.SECTION_GRID)); + var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID)); try { return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class); } catch (IOException e) { @@ -95,8 +95,8 @@ public class RedactionStorageService { public static class StorageIdUtils { - public static String getStorageId(String projectId, String fileId, FileType fileType) { - return projectId + "/" + fileId + "." + fileType.name() + fileType.getExtension(); + public static String getStorageId(String dossierId, String fileId, FileType fileType) { + return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 6f14d819..c7f8a107 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -128,8 +128,8 @@ public class RedactionIntegrationTest { private final Colors colors = new Colors(); private final Map reanlysisVersions = new HashMap<>(); - private final static String TEST_RULESET_ID = "123"; - private final static String TEST_PROJECT_ID = "123"; + private final static String TEST_DOSSIER_TEMPLATE_ID = "123"; + private final static String TEST_DOSSIER_ID = "123"; private final static String TEST_FILE_ID = "123"; @Configuration @@ -173,21 +173,21 @@ public class RedactionIntegrationTest { @Before public void stubClients() { - when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(0L); - when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new RulesResponse(RULES)); loadDictionaryForTest(); loadTypeForTest(); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(0L); - when(dictionaryClient.getAllTypes(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse.builder() + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(0L); + when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse.builder() .types(getTypeResponse()) .build()); - when(dictionaryClient.getVersion(TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypes(TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(TypeResponse.builder() + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(TypeResponse.builder() .types(List.of(TypeResult.builder() .type(DOSSIER_REDACTIONS) - .ruleSetId(TEST_RULESET_ID) + .ruleSetId(TEST_DOSSIER_TEMPLATE_ID) .hexColor( "#ffe187") .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) @@ -196,28 +196,28 @@ public class RedactionIntegrationTest { .build())) .build()); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); - when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(ADDRESS, false)); - when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(AUTHOR, false)); - when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SPONSOR, false)); - when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); - when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); - when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(HINT_ONLY, false)); - when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); - when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); - when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(TEST_METHOD, false)); - when(dictionaryClient.getDictionaryForType(PII, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PII, false)); - when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false)); - when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false)); - when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); - when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PURITY, false)); - when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(IMAGE, false)); - when(dictionaryClient.getDictionaryForType(OCR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(OCR, false)); - when(dictionaryClient.getDictionaryForType(LOGO, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(LOGO, false)); - when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SIGNATURE, false)); - when(dictionaryClient.getDictionaryForType(FORMULA, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FORMULA, false)); - when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS, TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true)); - when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors); + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SPONSOR, false)); + when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(HINT_ONLY, false)); + when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); + when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); + when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(TEST_METHOD, false)); + when(dictionaryClient.getDictionaryForType(PII, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PII, false)); + when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); + when(dictionaryClient.getDictionaryForType(PURITY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PURITY, false)); + when(dictionaryClient.getDictionaryForType(IMAGE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(IMAGE, false)); + when(dictionaryClient.getDictionaryForType(OCR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(OCR, false)); + when(dictionaryClient.getDictionaryForType(LOGO, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(LOGO, false)); + when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SIGNATURE, false)); + when(dictionaryClient.getDictionaryForType(FORMULA, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FORMULA, false)); + when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS, TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true)); + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } @@ -461,7 +461,7 @@ public class RedactionIntegrationTest { .stream() .map(typeColor -> TypeResult.builder() .type(typeColor.getKey()) - .ruleSetId(TEST_RULESET_ID) + .ruleSetId(TEST_DOSSIER_TEMPLATE_ID) .hexColor(typeColor.getValue()) .isHint(hintTypeMap.get(typeColor.getKey())) .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) @@ -525,7 +525,7 @@ public class RedactionIntegrationTest { Map> duplicates = new HashMap<>(); - var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); redactionLog.getRedactionLogEntry().forEach(entry -> { duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); @@ -536,10 +536,10 @@ public class RedactionIntegrationTest { }); dictionary.get(AUTHOR).add("Drinking water"); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L); AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() - .projectId(TEST_PROJECT_ID) + .dossierId(TEST_DOSSIER_ID) .fileId(TEST_FILE_ID) .build()); @@ -581,7 +581,7 @@ public class RedactionIntegrationTest { Map> duplicates = new HashMap<>(); - var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); redactionLog.getRedactionLogEntry().forEach(entry -> { duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); @@ -592,7 +592,7 @@ public class RedactionIntegrationTest { }); dictionary.get(AUTHOR).add("Drinking water"); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L); long rstart = System.currentTimeMillis(); reanalyzeService.reanalyze(request); @@ -636,8 +636,8 @@ public class RedactionIntegrationTest { AnalyzeResult result = reanalyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); - var text = redactionStorageService.getText(TEST_PROJECT_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID); redactionLog.getRedactionLogEntry().forEach(entry -> { if (entry.isImage()) { @@ -650,7 +650,7 @@ public class RedactionIntegrationTest { System.out.println("first analysis duration: " + (end - start)); try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) { - fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_PROJECT_ID, TEST_FILE_ID))); + fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID))); } int correctFound = 0; @@ -683,9 +683,9 @@ public class RedactionIntegrationTest { dictionary.get(VERTEBRATE).add("s-metolachlor"); reanlysisVersions.put("s-metolachlor", 3L); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); start = System.currentTimeMillis(); AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request); @@ -694,7 +694,7 @@ public class RedactionIntegrationTest { System.out.println("reanalysis analysis duration: " + (end - start)); AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() - .projectId(TEST_PROJECT_ID) + .dossierId(TEST_DOSSIER_ID) .fileId(TEST_FILE_ID) .build()); @@ -715,7 +715,7 @@ public class RedactionIntegrationTest { AnalyzeResult result = reanalyzeService.analyze(request); AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() - .projectId(TEST_PROJECT_ID) + .dossierId(TEST_DOSSIER_ID) .fileId(TEST_FILE_ID) .build()); @@ -786,7 +786,7 @@ public class RedactionIntegrationTest { AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() - .projectId(TEST_PROJECT_ID) + .dossierId(TEST_DOSSIER_ID) .fileId(TEST_FILE_ID) .build()); @@ -811,9 +811,9 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); RedactionRequest redactionRequest = RedactionRequest.builder() - .projectId(request.getProjectId()) + .dossierId(request.getDossierId()) .fileId(request.getFileId()) - .ruleSetId(request.getRuleSetId()) + .dossierTemplateId(request.getDossierTemplateId()) .build(); RedactionResult result = redactionController.classify(redactionRequest); @@ -833,9 +833,9 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); RedactionRequest redactionRequest = RedactionRequest.builder() - .projectId(request.getProjectId()) + .dossierId(request.getDossierId()) .fileId(request.getFileId()) - .ruleSetId(request.getRuleSetId()) + .dossierTemplateId(request.getDossierTemplateId()) .build(); RedactionResult result = redactionController.sections(redactionRequest); @@ -855,9 +855,9 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); RedactionRequest redactionRequest = RedactionRequest.builder() - .projectId(request.getProjectId()) + .dossierId(request.getDossierId()) .fileId(request.getFileId()) - .ruleSetId(request.getRuleSetId()) + .dossierTemplateId(request.getDossierTemplateId()) .build(); RedactionResult result = redactionController.htmlTables(redactionRequest); @@ -877,9 +877,9 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); RedactionRequest redactionRequest = RedactionRequest.builder() - .projectId(request.getProjectId()) + .dossierId(request.getDossierId()) .fileId(request.getFileId()) - .ruleSetId(request.getRuleSetId()) + .dossierTemplateId(request.getDossierTemplateId()) .build(); RedactionResult result = redactionController.htmlTables(redactionRequest); @@ -899,7 +899,7 @@ public class RedactionIntegrationTest { AnalyzeResult result = reanalyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); redactionLog.getRedactionLogEntry().forEach(entry -> { if (!entry.isHint()) { @@ -920,15 +920,15 @@ public class RedactionIntegrationTest { private AnalyzeRequest prepareStorage(InputStream stream) { AnalyzeRequest request = AnalyzeRequest.builder() - .ruleSetId(TEST_RULESET_ID) - .projectId(TEST_PROJECT_ID) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .dossierId(TEST_DOSSIER_ID) .fileId(TEST_FILE_ID) .lastProcessed(OffsetDateTime.now()) .build(); var bytes = IOUtils.toByteArray(stream); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_PROJECT_ID, TEST_FILE_ID, FileType.ORIGIN), bytes); + storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes); return request; @@ -947,7 +947,7 @@ public class RedactionIntegrationTest { AnalyzeResult result = reanalyzeService.analyze(request); AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() - .projectId(TEST_PROJECT_ID) + .dossierId(TEST_DOSSIER_ID) .fileId(TEST_FILE_ID) .build()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java index 512710d0..e4afe9d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java @@ -82,7 +82,7 @@ public class EntityRedactionServiceTest { @MockBean private LegalBasisClient legalBasisClient; - private final static String TEST_RULESET_ID = "123"; + private final static String TEST_DOSSIER_TEMPLATE_ID = "123"; @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @@ -138,20 +138,20 @@ public class EntityRedactionServiceTest { DictionaryResponse dictionaryResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))) .build(); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); DictionaryResponse addressResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse sponsorResponse = DictionaryResponse.builder() .entries(Collections.emptyList()) .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities } @@ -165,19 +165,19 @@ public class EntityRedactionServiceTest { DictionaryResponse dictionaryResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))) .build(); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); DictionaryResponse addressResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse sponsorResponse = DictionaryResponse.builder() .entries(Collections.emptyList()) .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities } @@ -188,21 +188,21 @@ public class EntityRedactionServiceTest { ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" + " Supplement - Identity of the active substance - Reference list.pdf"); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); DictionaryResponse dictionaryResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))) .build(); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); DictionaryResponse addressResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse sponsorResponse = DictionaryResponse.builder() .entries(Collections.emptyList()) .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities() .entrySet() .stream() @@ -210,7 +210,7 @@ public class EntityRedactionServiceTest { pdfFileResource = new ClassPathResource("files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of " + "the plant protection product.pdf"); classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities() .entrySet() .stream() @@ -221,21 +221,21 @@ public class EntityRedactionServiceTest { public void testFalsePositiveInWrongCell() throws IOException { ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf"); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); DictionaryResponse dictionaryResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))) .build(); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); DictionaryResponse addressResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse sponsorResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))) .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream() .filter(entity -> entity.getMatchedRule() == 9) @@ -283,26 +283,26 @@ public class EntityRedactionServiceTest { "\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " section.redactBetween(\"Contact:\", \"Tel.:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " end"; - when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet()); - when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules)); - droolsExecutionService.updateRules(TEST_RULESET_ID); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(RULES_VERSION.incrementAndGet()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new RulesResponse(tableRules)); + droolsExecutionService.updateRules(TEST_DOSSIER_TEMPLATE_ID); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf"); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); DictionaryResponse dictionaryResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))) .build(); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); DictionaryResponse addressResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse sponsorResponse = DictionaryResponse.builder() .entries(Collections.emptyList()) .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream() .filter(entity -> entity.getMatchedRule() == 6) @@ -322,26 +322,26 @@ public class EntityRedactionServiceTest { "Section(searchText.toLowerCase().contains(\"batches produced at\"))\n" + " then\n" + " section" + ".redactIfPrecededBy(\"batches produced at\", \"sponsor\", 11, \"Redacted because it represents a " + "sponsor company\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " end"; - when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet()); - when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules)); - droolsExecutionService.updateRules(TEST_RULESET_ID); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(RULES_VERSION.incrementAndGet()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new RulesResponse(tableRules)); + droolsExecutionService.updateRules(TEST_DOSSIER_TEMPLATE_ID); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf"); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); DictionaryResponse addressResponse = DictionaryResponse.builder() .entries(Collections.emptyList()) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse authorResponse = DictionaryResponse.builder() .entries(Collections.emptyList()) .build(); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(authorResponse); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(authorResponse); DictionaryResponse dictionaryResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))) .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream() .filter(entity -> entity.getMatchedRule() == 11) @@ -359,19 +359,19 @@ public class EntityRedactionServiceTest { .entries(toDictionaryEntry(Arrays.asList("Bissig R.", "Thanei P."))) .build(); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); DictionaryResponse addressResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse sponsorResponse = DictionaryResponse.builder() .entries(Collections.emptyList()) .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8); assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y @@ -382,15 +382,15 @@ public class EntityRedactionServiceTest { .entries(toDictionaryEntry(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C."))) .build(); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); addressResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9); @@ -407,19 +407,19 @@ public class EntityRedactionServiceTest { .entries(toDictionaryEntry(Collections.singletonList("Aldershof S."))) .build(); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); DictionaryResponse addressResponse = DictionaryResponse.builder() .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))) .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse sponsorResponse = DictionaryResponse.builder() .entries(Collections.emptyList()) .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6); } @@ -446,31 +446,31 @@ public class EntityRedactionServiceTest { "\"Yes\"))\n" + " then\n" + " section.redactCell(\"Author(s)\", 9, \"name\", false, \"Redacted because row is a vertebrate study\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + - " section.redact(\"address\", 9, \"Redacted because row is a vertebrate study\", \"Reg (EC) No" + + " section.redact(\"address\", 9, \"Redacted because row is a vertebrate sgitudy\", \"Reg (EC) No" + " 1107/2009 Art. 63 (2g)\");\n" + " section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" + " end"; - when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet()); - when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(RULES_VERSION.incrementAndGet()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new RulesResponse(tableRules)); TypeResponse typeResponse = TypeResponse.builder() .types(Arrays.asList( - TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(AUTHOR_CODE).hexColor("#ffff00").build(), - TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(ADDRESS_CODE).hexColor("#ff00ff").build(), - TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(SPONSOR_CODE).hexColor("#00ffff").build())) + TypeResult.builder().ruleSetId(TEST_DOSSIER_TEMPLATE_ID).type(AUTHOR_CODE).hexColor("#ffff00").build(), + TypeResult.builder().ruleSetId(TEST_DOSSIER_TEMPLATE_ID).type(ADDRESS_CODE).hexColor("#ff00ff").build(), + TypeResult.builder().ruleSetId(TEST_DOSSIER_TEMPLATE_ID).type(SPONSOR_CODE).hexColor("#00ffff").build())) .build(); - when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); - when(dictionaryClient.getAllTypes(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(typeResponse); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(typeResponse); // Default empty return to prevent NPEs DictionaryResponse dictionaryResponse = DictionaryResponse.builder() .build(); - when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); DictionaryResponse addressResponse = DictionaryResponse.builder() .build(); - when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); DictionaryResponse sponsorResponse = DictionaryResponse.builder() .build(); - when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Colors colors = new Colors(); colors.setDefaultColor("#acfc00"); @@ -478,7 +478,7 @@ public class EntityRedactionServiceTest { colors.setRequestAdd("#04b093"); colors.setRequestRemove("#04b093"); - when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors); + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); }