Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #162: migrated ruleset and projectid to new naming

Browse Source 
Merge in RED/redaction-service from RED-1329 to master

* commit 'ceaaed4fc721c9488582d5fce223a9ced928862a':
  migrated ruleset and projectid to new naming
  migrated ruleset and projectid to new naming
This commit is contained in:
Timo Bejan 2021-05-31 10:44:17 +02:00
commit 3995c9a3e7
21 changed files with 312 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java
View File

@ -13,9 +13,9 @@ import java.time.OffsetDateTime;
@AllArgsConstructor @AllArgsConstructor
public class AnalyzeRequest { public class AnalyzeRequest {
private String projectId; private String dossierId;
private String fileId; private String fileId;
private String ruleSetId; private String dossierTemplateId;
private boolean reanalyseOnlyIfPossible; private boolean reanalyseOnlyIfPossible;
private ManualRedactions manualRedactions; private ManualRedactions manualRedactions;
private OffsetDateTime lastProcessed; private OffsetDateTime lastProcessed;

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java
View File

@ -11,7 +11,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor @AllArgsConstructor
public class AnalyzeResult { public class AnalyzeResult {
private String projectId; private String dossierId;
private String fileId; private String fileId;
private long duration; private long duration;
private int numberOfPages; private int numberOfPages;

3
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnnotateRequest.java
View File

@ -11,6 +11,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor @AllArgsConstructor
public class AnnotateRequest { public class AnnotateRequest {
private String projectId; private String dossierId;
private String dossierTemplateId;
private String fileId; private String fileId;
} }

4
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java
View File

@ -15,8 +15,8 @@ public class RedactionChangeLog {
private List<RedactionChangeLogEntry> redactionLogEntry = new ArrayList<>(); private List<RedactionChangeLogEntry> redactionLogEntry = new ArrayList<>();
private long dictionaryVersion = -1; private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1; private long rulesVersion = -1;
private long legalBasisVersion = -1;
private String ruleSetId;
} }

8
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java
View File

@ -13,15 +13,9 @@ public class RedactionLog {
private List<RedactionLogEntry> redactionLogEntry; private List<RedactionLogEntry> redactionLogEntry;
private List<LegalBasisMapping> legalBasis; private List<LegalBasisMapping> legalBasis;
private String ruleSetId;
private long dictionaryVersion = -1; private long dictionaryVersion = -1;
private long rulesVersion = -1;
private long dossierDictionaryVersion = -1; private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1; private long legalBasisVersion = -1;
} }

4
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java
View File

@ -11,8 +11,8 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor @AllArgsConstructor
public class RedactionRequest { public class RedactionRequest {
private String projectId; private String dossierId;
private String fileId; private String fileId;
private String ruleSetId; private String dossierTemplateId;
private ManualRedactions manualRedactions; private ManualRedactions manualRedactions;
} }

4
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java
View File

@ -10,7 +10,7 @@ public interface RedactionResource {
String SERVICE_NAME = "redaction-service-v1"; String SERVICE_NAME = "redaction-service-v1";
String RULE_SET_PARAMETER_NAME = "ruleSetId"; String RULE_SET_PARAMETER_NAME = "dossierTemplateId";
String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}"; String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}";
@ -27,7 +27,7 @@ public interface RedactionResource {
RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest); RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
@PostMapping(value = "/rules/update" + RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE) @PostMapping(value = "/rules/update" + RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId); void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId);
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE) @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
void testRules(@RequestBody String rules); void testRules(@RequestBody String rules);

22
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -43,14 +43,14 @@ public class RedactionController implements RedactionResource {
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getProjectId(), annotateRequest.getFileId(), FileType.ORIGIN)); var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getProjectId(), annotateRequest.getFileId()); var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getProjectId(), annotateRequest.getFileId()); var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) { try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
pdDocument.setAllSecurityToBeRemoved(true); pdDocument.setAllSecurityToBeRemoved(true);
dictionaryService.updateDictionary(redactionLog.getRuleSetId(), annotateRequest.getProjectId()); dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
annotationService.annotate(pdDocument, redactionLog, sectionsGrid); annotationService.annotate(pdDocument, redactionLog, sectionsGrid);
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
@ -66,11 +66,11 @@ public class RedactionController implements RedactionResource {
@Override @Override
public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) { public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
try { try {
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream); Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) { try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
pdDocument.setAllSecurityToBeRemoved(true); pdDocument.setAllSecurityToBeRemoved(true);
@ -91,11 +91,11 @@ public class RedactionController implements RedactionResource {
@Override @Override
public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) { public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
try { try {
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream); Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) { try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
pdDocument.setAllSecurityToBeRemoved(true); pdDocument.setAllSecurityToBeRemoved(true);
@ -120,7 +120,7 @@ public class RedactionController implements RedactionResource {
Document classifiedDoc; Document classifiedDoc;
try { try {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getProjectId(), redactionRequest.getFileId(), FileType.ORIGIN)); var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, true); classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, true);
} catch (Exception e) { } catch (Exception e) {
throw new RedactionException(e); throw new RedactionException(e);
@ -143,9 +143,9 @@ public class RedactionController implements RedactionResource {
@Override @Override
public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId) { public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId) {
droolsExecutionService.updateRules(ruleSetId); droolsExecutionService.updateRules(dossierTemplateId);
} }

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java
View File

@ -38,7 +38,7 @@ public class RedactionMessageReceiver {
} }
log.info("Successfully analyzed {}", analyzeRequest); log.info("Successfully analyzed {}", analyzeRequest);
fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), result); fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
} }
@RabbitHandler @RabbitHandler
@ -48,7 +48,7 @@ public class RedactionMessageReceiver {
var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class); var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
log.info("Failed to process analyze request: {}", analyzeRequest); log.info("Failed to process analyze request: {}", analyzeRequest);
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getProjectId(), analyzeRequest.getFileId()); fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
} }
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java
View File

@ -10,7 +10,7 @@ import java.util.Map;
@Data @Data
public class DictionaryRepresentation { public class DictionaryRepresentation {
private String ruleSetId; private String dossierTemplateId;
private long dictionaryVersion = -1; private long dictionaryVersion = -1;
private List<DictionaryModel> dictionary = new ArrayList<>(); private List<DictionaryModel> dictionary = new ArrayList<>();
private float[] defaultColor; private float[] defaultColor;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java
View File

@ -11,6 +11,6 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor @AllArgsConstructor
public class DictionaryVersion { public class DictionaryVersion {
long rulesetVersion; long dossierTemplateVersion;
long dossierVersion; long dossierVersion;
} }

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java
View File

@ -9,7 +9,7 @@ import org.springframework.stereotype.Service;
@Service @Service
public class AnalyzeResponseService { public class AnalyzeResponseService {
public AnalyzeResult createAnalyzeResponse(String projectId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) {
boolean hasHints = redactionLog.getRedactionLogEntry().stream().anyMatch(RedactionLogEntry::isHint); boolean hasHints = redactionLog.getRedactionLogEntry().stream().anyMatch(RedactionLogEntry::isHint);
boolean hasRequests = redactionLog.getRedactionLogEntry() boolean hasRequests = redactionLog.getRedactionLogEntry()
@ -31,7 +31,7 @@ public class AnalyzeResponseService {
.isEmpty() && redactionChangeLog.getRedactionLogEntry().stream().anyMatch(entry -> !entry.getType().equals("false_positive")); .isEmpty() && redactionChangeLog.getRedactionLogEntry().stream().anyMatch(entry -> !entry.getType().equals("false_positive"));
return AnalyzeResult.builder() return AnalyzeResult.builder()
.projectId(projectId) .dossierId(dossierId)
.fileId(fileId) .fileId(fileId)
.duration(duration) .duration(duration)
.numberOfPages(pageCount) .numberOfPages(pageCount)

95
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -1,19 +1,12 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import static com.iqser.red.service.configuration.v1.api.resource.DictionaryResource.GLOBAL_DOSSIER;
import com.iqser.red.service.configuration.v1.api.model.Colors; import com.iqser.red.service.configuration.v1.api.model.Colors;
import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry; import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
import com.iqser.red.service.configuration.v1.api.model.TypeResponse; import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
import com.iqser.red.service.configuration.v1.api.model.TypeResult; import com.iqser.red.service.configuration.v1.api.model.TypeResult;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import feign.FeignException; import feign.FeignException;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -25,6 +18,8 @@ import java.awt.Color;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static com.iqser.red.service.configuration.v1.api.resource.DictionaryResource.GLOBAL_DOSSIER;
@Slf4j @Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@ -32,37 +27,37 @@ public class DictionaryService {
private final DictionaryClient dictionaryClient; private final DictionaryClient dictionaryClient;
private final Map<String, DictionaryRepresentation> dictionariesByRuleSets = new HashMap<>(); private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>(); private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
public DictionaryVersion updateDictionary(String ruleSetId, String dossierId) { public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
long rulesetDictionaryVersion = dictionaryClient.getVersion(ruleSetId, GLOBAL_DOSSIER); long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER);
var rulesetDictionary = dictionariesByRuleSets.get(ruleSetId); var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
if (rulesetDictionary == null || rulesetDictionaryVersion > rulesetDictionary.getDictionaryVersion()) { if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
updateDictionaryEntry(ruleSetId, rulesetDictionaryVersion, GLOBAL_DOSSIER); updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, GLOBAL_DOSSIER);
} }
long dossierDictionaryVersion = dictionaryClient.getVersion(ruleSetId, dossierId); long dossierDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, dossierId);
var dossierDictionary = dictionariesByDossier.get(dossierId); var dossierDictionary = dictionariesByDossier.get(dossierId);
if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) { if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
updateDictionaryEntry(ruleSetId, dossierDictionaryVersion, dossierId); updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, dossierId);
} }
return DictionaryVersion.builder().rulesetVersion(rulesetDictionaryVersion).dossierVersion(dossierDictionaryVersion).build(); return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
} }
public DictionaryIncrement getDictionaryIncrements(String ruleSetId, DictionaryVersion fromVersion, String dossierId) { public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
DictionaryVersion version = updateDictionary(ruleSetId, dossierId); DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
Set<DictionaryIncrementValue> newValues = new HashSet<>(); Set<DictionaryIncrementValue> newValues = new HashSet<>();
List<DictionaryModel> dictionaryModels = dictionariesByRuleSets.get(ruleSetId).getDictionary(); List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> { dictionaryModels.forEach(dictionaryModel -> {
dictionaryModel.getEntries().forEach(dictionaryEntry -> { dictionaryModel.getEntries().forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getRulesetVersion()) { if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
} }
}); });
@ -83,12 +78,12 @@ public class DictionaryService {
} }
private void updateDictionaryEntry(String ruleSetId, long version, String dossierId) { private void updateDictionaryEntry(String dossierTemplateId, long version, String dossierId) {
try { try {
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation(); DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
TypeResponse typeResponse = dictionaryClient.getAllTypes(ruleSetId, dossierId); TypeResponse typeResponse = dictionaryClient.getAllTypes(dossierTemplateId, dossierId);
if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) { if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
List<DictionaryModel> dictionary = typeResponse.getTypes() List<DictionaryModel> dictionary = typeResponse.getTypes()
@ -100,18 +95,18 @@ public class DictionaryService {
dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm)); dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
Colors colors = dictionaryClient.getColors(ruleSetId); Colors colors = dictionaryClient.getColors(dossierTemplateId);
dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor())); dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd())); dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove())); dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted())); dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
dictionaryRepresentation.setRuleSetId(ruleSetId); dictionaryRepresentation.setDossierTemplateId(dossierTemplateId);
dictionaryRepresentation.setDictionaryVersion(version); dictionaryRepresentation.setDictionaryVersion(version);
dictionaryRepresentation.setDictionary(dictionary); dictionaryRepresentation.setDictionary(dictionary);
if(dossierId.equals(GLOBAL_DOSSIER)) { if(dossierId.equals(GLOBAL_DOSSIER)) {
dictionariesByRuleSets.put(ruleSetId, dictionaryRepresentation); dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
} else { } else {
dictionariesByDossier.put(dossierId, dictionaryRepresentation); dictionariesByDossier.put(dossierId, dictionaryRepresentation);
} }
@ -123,14 +118,14 @@ public class DictionaryService {
} }
public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) { public void updateExternalDictionary(Dictionary dictionary, String dossierTemplateId) {
dictionary.getDictionaryModels().forEach(dm -> { dictionary.getDictionaryModels().forEach(dm -> {
if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) { if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false, GLOBAL_DOSSIER); dictionaryClient.addEntries(dm.getType(), dossierTemplateId, new ArrayList<>(dm.getLocalEntries()), false, GLOBAL_DOSSIER);
long externalVersion = dictionaryClient.getVersion(ruleSetId, GLOBAL_DOSSIER); long externalVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER);
if (externalVersion == dictionary.getVersion().getRulesetVersion() + 1) { if (externalVersion == dictionary.getVersion().getDossierTemplateVersion() + 1) {
dictionary.getVersion().setRulesetVersion(externalVersion); dictionary.getVersion().setDossierTemplateVersion(externalVersion);
} }
} }
}); });
@ -156,9 +151,9 @@ public class DictionaryService {
} }
public boolean isCaseInsensitiveDictionary(String type, String ruleSetId) { public boolean isCaseInsensitiveDictionary(String type, String dossierTemplateId) {
DictionaryModel dictionaryModel = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type); DictionaryModel dictionaryModel = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
if (dictionaryModel != null) { if (dictionaryModel != null) {
return dictionaryModel.isCaseInsensitive(); return dictionaryModel.isCaseInsensitive();
} }
@ -166,19 +161,19 @@ public class DictionaryService {
} }
public float[] getColor(String type, String ruleSetId) { public float[] getColor(String type, String dossierTemplateId) {
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type); DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
if (model != null) { if (model != null) {
return model.getColor(); return model.getColor();
} }
return dictionariesByRuleSets.get(ruleSetId).getDefaultColor(); return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
} }
public boolean isHint(String type, String ruleSetId) { public boolean isHint(String type, String dossierTemplateId) {
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type); DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
if (model != null) { if (model != null) {
return model.isHint(); return model.isHint();
} }
@ -186,9 +181,9 @@ public class DictionaryService {
} }
public boolean isRecommendation(String type, String ruleSetId) { public boolean isRecommendation(String type, String dossierTemplateId) {
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type); DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
if (model != null) { if (model != null) {
return model.isRecommendation(); return model.isRecommendation();
} }
@ -196,12 +191,12 @@ public class DictionaryService {
} }
public Dictionary getDeepCopyDictionary(String ruleSetId, String dossierId) { public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
List<DictionaryModel> copy = new ArrayList<>(); List<DictionaryModel> copy = new ArrayList<>();
var rulesetRepresentation = dictionariesByRuleSets.get(ruleSetId); var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
rulesetRepresentation.getDictionary().forEach(dm -> { dossierTemplateRepresentation.getDictionary().forEach(dm -> {
copy.add(SerializationUtils.clone(dm)); copy.add(SerializationUtils.clone(dm));
}); });
@ -215,25 +210,25 @@ public class DictionaryService {
dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
} }
return new Dictionary(copy, DictionaryVersion.builder().rulesetVersion(rulesetRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); return new Dictionary(copy, DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
} }
public float[] getRequestRemoveColor(String ruleSetId) { public float[] getRequestRemoveColor(String dossierTemplateId) {
return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor(); return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
} }
public float[] getNotRedactedColor(String ruleSetId) { public float[] getNotRedactedColor(String dossierTemplateId) {
return dictionariesByRuleSets.get(ruleSetId).getNotRedactedColor(); return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
} }
public float[] getRequestAddColor(String ruleSetId) { public float[] getRequestAddColor(String dossierTemplateId) {
return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor(); return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
} }
} }

38
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -28,14 +28,14 @@ public class DroolsExecutionService {
private final Map<String, KieContainer> kieContainers = new HashMap<>(); private final Map<String, KieContainer> kieContainers = new HashMap<>();
private final Map<String, Long> rulesVersionPerRuleSetId = new HashMap<>(); private final Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
public KieContainer getKieContainer(String ruleSetId) { public KieContainer getKieContainer(String dossierTemplateId) {
KieContainer container = kieContainers.get(ruleSetId); KieContainer container = kieContainers.get(dossierTemplateId);
if (container == null) { if (container == null) {
return createOrUpdateKieContainer(ruleSetId); return createOrUpdateKieContainer(dossierTemplateId);
} else { } else {
return container; return container;
} }
@ -55,43 +55,43 @@ public class DroolsExecutionService {
} }
public KieContainer updateRules(String ruleSetId) { public KieContainer updateRules(String dossierTemplateId) {
long version = rulesClient.getVersion(ruleSetId); long version = rulesClient.getVersion(dossierTemplateId);
Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId); Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
if (rulesVersion == null) { if (rulesVersion == null) {
rulesVersion = -1L; rulesVersion = -1L;
} }
if (version > rulesVersion.longValue()) { if (version > rulesVersion.longValue()) {
rulesVersionPerRuleSetId.put(ruleSetId, version); rulesVersionPerDossierTemplateId.put(dossierTemplateId, version);
return createOrUpdateKieContainer(ruleSetId); return createOrUpdateKieContainer(dossierTemplateId);
} }
return getKieContainer(ruleSetId); return getKieContainer(dossierTemplateId);
} }
private KieContainer createOrUpdateKieContainer(String ruleSetId) { private KieContainer createOrUpdateKieContainer(String dossierTemplateId) {
try { try {
RulesResponse rules = rulesClient.getRules(ruleSetId); RulesResponse rules = rulesClient.getRules(dossierTemplateId);
if (rules == null || StringUtils.isEmpty(rules.getRules())) { if (rules == null || StringUtils.isEmpty(rules.getRules())) {
throw new RuntimeException("Rules cannot be empty."); throw new RuntimeException("Rules cannot be empty.");
} }
KieServices kieServices = KieServices.Factory.get(); KieServices kieServices = KieServices.Factory.get();
KieModule kieModule = getKieModule(ruleSetId, rules.getRules(), kieServices); KieModule kieModule = getKieModule(dossierTemplateId, rules.getRules(), kieServices);
var container = kieContainers.get(ruleSetId); var container = kieContainers.get(dossierTemplateId);
if (container != null) { if (container != null) {
container.updateToVersion(kieModule.getReleaseId()); container.updateToVersion(kieModule.getReleaseId());
return container; return container;
} }
container = kieServices.newKieContainer(kieModule.getReleaseId()); container = kieServices.newKieContainer(kieModule.getReleaseId());
kieContainers.put(ruleSetId, container); kieContainers.put(dossierTemplateId, container);
return container; return container;
} catch (Exception e) { } catch (Exception e) {
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e); throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
@ -100,11 +100,11 @@ public class DroolsExecutionService {
} }
private KieModule getKieModule(String ruleSetId, String rules, KieServices kieServices) { private KieModule getKieModule(String dossierTemplateId, String rules, KieServices kieServices) {
KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8)); InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/main/resources/drools/rules" + ruleSetId + ".drl", kieServices.getResources() kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources()
.newInputStreamResource(input)); .newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll(); kieBuilder.buildAll();
@ -122,9 +122,9 @@ public class DroolsExecutionService {
} }
public long getRulesVersion(String ruleSetId) { public long getRulesVersion(String dossierTemplateId) {
Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId); Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
if (rulesVersion == null) { if (rulesVersion == null) {
return -1; return -1;
} }

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -33,13 +33,13 @@ public class EntityRedactionService {
private final SurroundingWordsService surroundingWordsService; private final SurroundingWordsService surroundingWordsService;
public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions, String dossierId) { public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId) {
dictionaryService.updateDictionary(ruleSetId, dossierId); dictionaryService.updateDictionary(dossierTemplateId, dossierId);
KieContainer container = droolsExecutionService.updateRules(ruleSetId); KieContainer container = droolsExecutionService.updateRules(dossierTemplateId);
long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId); long rulesVersion = droolsExecutionService.getRulesVersion(dossierTemplateId);
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId, dossierId); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(dossierTemplateId, dossierId);
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null)); Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
@ -75,7 +75,7 @@ public class EntityRedactionService {
} }
} }
dictionaryService.updateExternalDictionary(dictionary, ruleSetId); dictionaryService.updateExternalDictionary(dictionary, dossierTemplateId);
classifiedDoc.setDictionaryVersion(dictionary.getVersion()); classifiedDoc.setDictionaryVersion(dictionary.getVersion());
classifiedDoc.setRulesVersion(rulesVersion); classifiedDoc.setRulesVersion(rulesVersion);

57
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java
View File

@ -50,7 +50,7 @@ public class ReanalyzeService {
try { try {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest
.getProjectId(), analyzeRequest.getFileId(), FileType.ORIGIN)); .getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream); classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
pageCount = classifiedDoc.getPages().size(); pageCount = classifiedDoc.getPages().size();
} catch (Exception e) { } catch (Exception e) {
@ -58,35 +58,34 @@ public class ReanalyzeService {
} }
log.info("Document structure analysis successful, starting redaction analysis..."); log.info("Document structure analysis successful, starting redaction analysis...");
entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getRuleSetId(), analyzeRequest.getManualRedactions(), analyzeRequest entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest
.getProjectId()); .getDossierId());
redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest
.getRuleSetId()); .getDossierTemplateId());
log.info("Redaction analysis successful..."); log.info("Redaction analysis successful...");
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getRuleSetId()); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis, var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis,
analyzeRequest.getRuleSetId(), classifiedDoc.getDictionaryVersion().getDossierTemplateVersion(),
classifiedDoc.getDictionaryVersion().getRulesetVersion(),
classifiedDoc.getRulesVersion(),
classifiedDoc.getDictionaryVersion().getDossierVersion(), classifiedDoc.getDictionaryVersion().getDossierVersion(),
legalBasisClient.getVersion(analyzeRequest.getRuleSetId())); classifiedDoc.getRulesVersion(),
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
log.info("Analyzed with rules {} and dictionary {} for ruleSet: {}", classifiedDoc.getRulesVersion(), classifiedDoc log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc
.getDictionaryVersion(), analyzeRequest.getRuleSetId()); .getDictionaryVersion(), analyzeRequest.getDossierTemplateId());
// first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data // first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data
var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), redactionLog); var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
// store redactionLog // store redactionLog
redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc
.getSectionText())); .getSectionText()));
redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
.getSectionGrid()); .getSectionGrid());
long duration = System.currentTimeMillis() - startTime; long duration = System.currentTimeMillis() - startTime;
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog); return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog);
} }
@ -95,16 +94,16 @@ public class ReanalyzeService {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId()); var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
var text = redactionStorageService.getText(analyzeRequest.getProjectId(), analyzeRequest.getFileId()); var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
// not yet ready for reanalysis // not yet ready for reanalysis
if (redactionLog == null || text == null || text.getNumberOfPages() == 0) { if (redactionLog == null || text == null || text.getNumberOfPages() == 0) {
return analyze(analyzeRequest); return analyze(analyzeRequest);
} }
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getRuleSetId(), new DictionaryVersion(redactionLog DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog
.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getProjectId()); .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions()); Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions());
Map<String, List<Comment>> comments = null; Map<String, List<Comment>> comments = null;
@ -164,9 +163,9 @@ public class ReanalyzeService {
//-- //--
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getRuleSetId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getRuleSetId(), analyzeRequest.getProjectId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>(); List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
for (SectionText reanalysisSection : reanalysisSections) { for (SectionText reanalysisSection : reanalysisSections) {
@ -229,16 +228,16 @@ public class ReanalyzeService {
for (int page = 1; page <= text.getNumberOfPages(); page++) { for (int page = 1; page <= text.getNumberOfPages(); page++) {
if (entitiesPerPage.get(page) != null) { if (entitiesPerPage.get(page) != null) {
newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
.getRuleSetId())); .getDossierTemplateId()));
} }
if (imagesPerPage.get(page) != null) { if (imagesPerPage.get(page) != null) {
newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
.getRuleSetId())); .getDossierTemplateId()));
} }
newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest
.getRuleSetId())); .getDossierTemplateId()));
} }
redactionLog.getRedactionLogEntry() redactionLog.getRedactionLogEntry()
@ -253,15 +252,15 @@ public class ReanalyzeService {
RedactionLog redactionLog, Text text, RedactionLog redactionLog, Text text,
DictionaryIncrement dictionaryIncrement) { DictionaryIncrement dictionaryIncrement) {
redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getRulesetVersion()); redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierTemplateVersion());
redactionLog.setDossierDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierVersion()); redactionLog.setDossierDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierVersion());
var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), redactionLog); var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
long duration = System.currentTimeMillis() - startTime; long duration = System.currentTimeMillis() - startTime;
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), duration, text return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text
.getNumberOfPages(), redactionLog, changeLog); .getNumberOfPages(), redactionLog, changeLog);
} }

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java
View File

@ -22,12 +22,12 @@ public class RedactionChangeLogService {
private final RedactionStorageService redactionStorageService; private final RedactionStorageService redactionStorageService;
public RedactionChangeLog createAndStoreChangeLog(String projectId, String fileId, RedactionLog currentRedactionLog) { public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) {
try { try {
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(projectId, fileId); RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId);
var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog); var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog);
redactionStorageService.storeObject(projectId, fileId, FileType.REDACTION_CHANGELOG, changeLog); redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog);
return changeLog; return changeLog;
} catch (Exception e) { } catch (Exception e) {
log.debug("Previous redaction log not available"); log.debug("Previous redaction log not available");
@ -57,8 +57,11 @@ public class RedactionChangeLogService {
.map(entry -> convert(entry, ChangeType.REMOVED)) .map(entry -> convert(entry, ChangeType.REMOVED))
.collect(Collectors.toList())); .collect(Collectors.toList()));
return new RedactionChangeLog(changeLogEntries, currentRedactionLog.getDictionaryVersion(), currentRedactionLog.getRulesVersion(), currentRedactionLog return new RedactionChangeLog(changeLogEntries,
.getRuleSetId()); currentRedactionLog.getDictionaryVersion(),
currentRedactionLog.getDossierDictionaryVersion(),
currentRedactionLog.getRulesVersion(),
currentRedactionLog.getLegalBasisVersion());
} }

90
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -32,7 +32,7 @@ public class RedactionLogCreatorService {
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions, public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
String ruleSetId) { String dossierTemplateId) {
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions); Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
@ -42,24 +42,24 @@ public class RedactionLogCreatorService {
if (classifiedDoc.getEntities().get(page) != null) { if (classifiedDoc.getEntities().get(page) != null) {
classifiedDoc.getRedactionLogEntities() classifiedDoc.getRedactionLogEntities()
.addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, ruleSetId)); .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId));
} }
if (manualRedactionPages.contains(page)) { if (manualRedactionPages.contains(page)) {
classifiedDoc.getRedactionLogEntities() classifiedDoc.getRedactionLogEntities()
.addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, ruleSetId)); .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId));
} }
if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) { if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) {
classifiedDoc.getRedactionLogEntities() classifiedDoc.getRedactionLogEntities()
.addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, ruleSetId)); .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId));
} }
} }
} }
public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, ManualRedactions manualRedactions, public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, ManualRedactions manualRedactions,
int pageNumber, String ruleSetId) { int pageNumber, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>(); List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -69,14 +69,14 @@ public class RedactionLogCreatorService {
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
.id(id) .id(id)
.color(getColorForImage(image, ruleSetId, false)) .color(getColorForImage(image, dossierTemplateId, false))
.isImage(true) .isImage(true)
.type(image.getType()) .type(image.getType())
.redacted(image.isRedaction()) .redacted(image.isRedaction())
.reason(image.getRedactionReason()) .reason(image.getRedactionReason())
.legalBasis(image.getLegalBasis()) .legalBasis(image.getLegalBasis())
.matchedRule(image.getMatchedRule()) .matchedRule(image.getMatchedRule())
.isHint(dictionaryService.isHint(image.getType(), ruleSetId)) .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId))
.manual(false) .manual(false)
.isDictionaryEntry(false) .isDictionaryEntry(false)
.isRecommendation(false) .isRecommendation(false)
@ -96,11 +96,11 @@ public class RedactionLogCreatorService {
redactionLogEntry.setRedacted(false); redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = image.getRedactionReason() + ", removed by manual override"; manualOverrideReason = image.getRedactionReason() + ", removed by manual override";
redactionLogEntry.setColor(getColorForImage(image, ruleSetId, false)); redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = image.getRedactionReason() + ", requested to remove"; manualOverrideReason = image.getRedactionReason() + ", requested to remove";
redactionLogEntry.setStatus(Status.REQUESTED); redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(image, ruleSetId, true)); redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true));
} else { } else {
redactionLogEntry.setStatus(Status.DECLINED); redactionLogEntry.setStatus(Status.DECLINED);
} }
@ -121,13 +121,13 @@ public class RedactionLogCreatorService {
image.setRedaction(true); image.setRedaction(true);
redactionLogEntry.setRedacted(true); redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColorForImage(image, ruleSetId, false)); redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false));
manualOverrideReason = image.getRedactionReason() + ", forced by manual override"; manualOverrideReason = image.getRedactionReason() + ", forced by manual override";
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = image.getRedactionReason() + ", requested to force redact"; manualOverrideReason = image.getRedactionReason() + ", requested to force redact";
redactionLogEntry.setStatus(Status.REQUESTED); redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(image, ruleSetId, true)); redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else { } else {
redactionLogEntry.setStatus(Status.DECLINED); redactionLogEntry.setStatus(Status.DECLINED);
@ -166,7 +166,7 @@ public class RedactionLogCreatorService {
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions, public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions,
int page, String ruleSetId) { int page, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>(); List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -180,7 +180,7 @@ public class RedactionLogCreatorService {
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, ruleSetId); RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId);
if (processedIds.contains(entityPositionSequence.getId())) { if (processedIds.contains(entityPositionSequence.getId())) {
// TODO refactor this outer loop jump as soon as we have the time. // TODO refactor this outer loop jump as soon as we have the time.
@ -199,11 +199,11 @@ public class RedactionLogCreatorService {
redactionLogEntry.setRedacted(false); redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = entity.getRedactionReason() + ", removed by manual override"; manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
redactionLogEntry.setColor(getColor(entity, ruleSetId, false)); redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = entity.getRedactionReason() + ", requested to remove"; manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
redactionLogEntry.setStatus(Status.REQUESTED); redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(entity, ruleSetId, true)); redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true));
} else { } else {
redactionLogEntry.setStatus(Status.DECLINED); redactionLogEntry.setStatus(Status.DECLINED);
} }
@ -224,13 +224,13 @@ public class RedactionLogCreatorService {
entity.setRedaction(true); entity.setRedaction(true);
redactionLogEntry.setRedacted(true); redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColor(entity, ruleSetId, false)); redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false));
manualOverrideReason = entity.getRedactionReason() + ", forced by manual override"; manualOverrideReason = entity.getRedactionReason() + ", forced by manual override";
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = entity.getRedactionReason() + ", requested to force redact"; manualOverrideReason = entity.getRedactionReason() + ", requested to force redact";
redactionLogEntry.setStatus(Status.REQUESTED); redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(entity, ruleSetId, true)); redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else { } else {
redactionLogEntry.setStatus(Status.DECLINED); redactionLogEntry.setStatus(Status.DECLINED);
@ -299,7 +299,7 @@ public class RedactionLogCreatorService {
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
Map<String, List<Comment>> comments, int page, Map<String, List<Comment>> comments, int page,
String ruleSetId) { String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>(); List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -311,7 +311,7 @@ public class RedactionLogCreatorService {
String id = manualRedactionEntry.getId(); String id = manualRedactionEntry.getId();
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, ruleSetId); RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId);
List<Rectangle> rectanglesOnPage = new ArrayList<>(); List<Rectangle> rectanglesOnPage = new ArrayList<>();
for (Rectangle rectangle : manualRedactionEntry.getPositions()) { for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
@ -338,11 +338,11 @@ public class RedactionLogCreatorService {
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
String ruleSetId) { String dossierTemplateId) {
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.id(id) .id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), ruleSetId, manualRedactionEntry.getStatus())) .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason()) .reason(manualRedactionEntry.getReason())
.legalBasis(manualRedactionEntry.getLegalBasis()) .legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue()) .value(manualRedactionEntry.getValue())
@ -360,17 +360,17 @@ public class RedactionLogCreatorService {
} }
private RedactionLogEntry createRedactionLogEntry(Entity entity, String ruleSetId) { private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) {
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.color(getColor(entity, ruleSetId, false)) .color(getColor(entity, dossierTemplateId, false))
.reason(entity.getRedactionReason()) .reason(entity.getRedactionReason())
.legalBasis(entity.getLegalBasis()) .legalBasis(entity.getLegalBasis())
.value(entity.getWord()) .value(entity.getWord())
.type(entity.getType()) .type(entity.getType())
.redacted(entity.isRedaction()) .redacted(entity.isRedaction())
.isHint(isHint(entity, ruleSetId)) .isHint(isHint(entity, dossierTemplateId))
.isRecommendation(isRecommendation(entity, ruleSetId)) .isRecommendation(isRecommendation(entity, dossierTemplateId))
.section(entity.getHeadline()) .section(entity.getHeadline())
.sectionNumber(entity.getSectionNumber()) .sectionNumber(entity.getSectionNumber())
.matchedRule(entity.getMatchedRule()) .matchedRule(entity.getMatchedRule())
@ -384,56 +384,56 @@ public class RedactionLogCreatorService {
} }
private float[] getColor(Entity entity, String ruleSetId, boolean requestedToRemove) { private float[] getColor(Entity entity, String dossierTemplateId, boolean requestedToRemove) {
if (requestedToRemove) { if (requestedToRemove) {
return dictionaryService.getRequestRemoveColor(ruleSetId); return dictionaryService.getRequestRemoveColor(dossierTemplateId);
} }
if (!entity.isRedaction() && !isHint(entity, ruleSetId)) { if (!entity.isRedaction() && !isHint(entity, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(ruleSetId); return dictionaryService.getNotRedactedColor(dossierTemplateId);
} }
return dictionaryService.getColor(entity.getType(), ruleSetId); return dictionaryService.getColor(entity.getType(), dossierTemplateId);
} }
private float[] getColorForManualAdd(String type, String ruleSetId, Status status) { private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) {
if (status.equals(Status.REQUESTED)) { if (status.equals(Status.REQUESTED)) {
return dictionaryService.getRequestAddColor(ruleSetId); return dictionaryService.getRequestAddColor(dossierTemplateId);
} else if (status.equals(Status.DECLINED)) { } else if (status.equals(Status.DECLINED)) {
return dictionaryService.getNotRedactedColor(ruleSetId); return dictionaryService.getNotRedactedColor(dossierTemplateId);
} }
return getColor(type, ruleSetId); return getColor(type, dossierTemplateId);
} }
private float[] getColor(String type, String ruleSetId) { private float[] getColor(String type, String dossierTemplateId) {
return dictionaryService.getColor(type, ruleSetId); return dictionaryService.getColor(type, dossierTemplateId);
} }
private float[] getColorForImage(Image image, String ruleSetId, boolean requestedToRemove) { private float[] getColorForImage(Image image, String dossierTemplateId, boolean requestedToRemove) {
if (requestedToRemove) { if (requestedToRemove) {
return dictionaryService.getRequestRemoveColor(ruleSetId); return dictionaryService.getRequestRemoveColor(dossierTemplateId);
} }
if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), ruleSetId)) { if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(ruleSetId); return dictionaryService.getNotRedactedColor(dossierTemplateId);
} }
return dictionaryService.getColor(image.getType(), ruleSetId); return dictionaryService.getColor(image.getType(), dossierTemplateId);
} }
private boolean isHint(Entity entity, String ruleSetId) { private boolean isHint(Entity entity, String dossierTemplateId) {
return dictionaryService.isHint(entity.getType(), ruleSetId); return dictionaryService.isHint(entity.getType(), dossierTemplateId);
} }
private boolean isRecommendation(Entity entity, String ruleSetId) { private boolean isRecommendation(Entity entity, String dossierTemplateId) {
return dictionaryService.isRecommendation(entity.getType(), ruleSetId); return dictionaryService.isRecommendation(entity.getType(), dossierTemplateId);
} }

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -32,16 +32,16 @@ public class RedactionStorageService {
@SneakyThrows @SneakyThrows
public void storeObject(String projectId, String fileId, FileType fileType, Object any) { public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
storageService.storeObject(StorageIdUtils.getStorageId(projectId, fileId, fileType), objectMapper.writeValueAsBytes(any)); storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any));
} }
public RedactionLog getRedactionLog(String projectId, String fileId) { public RedactionLog getRedactionLog(String dossierId, String fileId) {
InputStreamResource inputStreamResource; InputStreamResource inputStreamResource;
try { try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(projectId, fileId, FileType.REDACTION_LOG)); inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("Text not available."); log.debug("Text not available.");
return null; return null;
@ -55,11 +55,11 @@ public class RedactionStorageService {
} }
public Text getText(String projectId, String fileId) { public Text getText(String dossierId, String fileId) {
InputStreamResource inputStreamResource; InputStreamResource inputStreamResource;
try { try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(projectId, fileId, FileType.TEXT)); inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("Text not available."); log.debug("Text not available.");
return null; return null;
@ -73,9 +73,9 @@ public class RedactionStorageService {
} }
public SectionGrid getSectionGrid(String projectId, String fileId) { public SectionGrid getSectionGrid(String dossierId, String fileId) {
var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(projectId, fileId, FileType.SECTION_GRID)); var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
try { try {
return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class); return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class);
} catch (IOException e) { } catch (IOException e) {
@ -95,8 +95,8 @@ public class RedactionStorageService {
public static class StorageIdUtils { public static class StorageIdUtils {
public static String getStorageId(String projectId, String fileId, FileType fileType) { public static String getStorageId(String dossierId, String fileId, FileType fileType) {
return projectId + "/" + fileId + "." + fileType.name() + fileType.getExtension(); return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
} }
} }

116
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -128,8 +128,8 @@ public class RedactionIntegrationTest {
private final Colors colors = new Colors(); private final Colors colors = new Colors();
private final Map<String, Long> reanlysisVersions = new HashMap<>(); private final Map<String, Long> reanlysisVersions = new HashMap<>();
private final static String TEST_RULESET_ID = "123"; private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
private final static String TEST_PROJECT_ID = "123"; private final static String TEST_DOSSIER_ID = "123";
private final static String TEST_FILE_ID = "123"; private final static String TEST_FILE_ID = "123";
@Configuration @Configuration
@ -173,21 +173,21 @@ public class RedactionIntegrationTest {
@Before @Before
public void stubClients() { public void stubClients() {
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(0L); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(RULES)); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new RulesResponse(RULES));
loadDictionaryForTest(); loadDictionaryForTest();
loadTypeForTest(); loadTypeForTest();
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(0L);
when(dictionaryClient.getAllTypes(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse.builder() when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse.builder()
.types(getTypeResponse()) .types(getTypeResponse())
.build()); .build());
when(dictionaryClient.getVersion(TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypes(TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(TypeResponse.builder() when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(TypeResponse.builder()
.types(List.of(TypeResult.builder() .types(List.of(TypeResult.builder()
.type(DOSSIER_REDACTIONS) .type(DOSSIER_REDACTIONS)
.ruleSetId(TEST_RULESET_ID) .ruleSetId(TEST_DOSSIER_TEMPLATE_ID)
.hexColor( "#ffe187") .hexColor( "#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
@ -196,28 +196,28 @@ public class RedactionIntegrationTest {
.build())) .build()))
.build()); .build());
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(ADDRESS, false)); when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(ADDRESS, false));
when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(AUTHOR, false)); when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SPONSOR, false)); when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(HINT_ONLY, false)); when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(TEST_METHOD, false)); when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(PII, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PII, false)); when(dictionaryClient.getDictionaryForType(PII, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PII, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false)); when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false)); when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PURITY, false)); when(dictionaryClient.getDictionaryForType(PURITY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(IMAGE, false)); when(dictionaryClient.getDictionaryForType(IMAGE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(IMAGE, false));
when(dictionaryClient.getDictionaryForType(OCR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(OCR, false)); when(dictionaryClient.getDictionaryForType(OCR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(OCR, false));
when(dictionaryClient.getDictionaryForType(LOGO, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(LOGO, false)); when(dictionaryClient.getDictionaryForType(LOGO, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(LOGO, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SIGNATURE, false)); when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FORMULA, false)); when(dictionaryClient.getDictionaryForType(FORMULA, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS, TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true)); when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS, TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
} }
@ -461,7 +461,7 @@ public class RedactionIntegrationTest {
.stream() .stream()
.map(typeColor -> TypeResult.builder() .map(typeColor -> TypeResult.builder()
.type(typeColor.getKey()) .type(typeColor.getKey())
.ruleSetId(TEST_RULESET_ID) .ruleSetId(TEST_DOSSIER_TEMPLATE_ID)
.hexColor(typeColor.getValue()) .hexColor(typeColor.getValue())
.isHint(hintTypeMap.get(typeColor.getKey())) .isHint(hintTypeMap.get(typeColor.getKey()))
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
@ -525,7 +525,7 @@ public class RedactionIntegrationTest {
Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>(); Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getRedactionLogEntry().forEach(entry -> { redactionLog.getRedactionLogEntry().forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
@ -536,10 +536,10 @@ public class RedactionIntegrationTest {
}); });
dictionary.get(AUTHOR).add("Drinking water"); dictionary.get(AUTHOR).add("Drinking water");
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.projectId(TEST_PROJECT_ID) .dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.build()); .build());
@ -581,7 +581,7 @@ public class RedactionIntegrationTest {
Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>(); Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getRedactionLogEntry().forEach(entry -> { redactionLog.getRedactionLogEntry().forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
@ -592,7 +592,7 @@ public class RedactionIntegrationTest {
}); });
dictionary.get(AUTHOR).add("Drinking water"); dictionary.get(AUTHOR).add("Drinking water");
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L);
long rstart = System.currentTimeMillis(); long rstart = System.currentTimeMillis();
reanalyzeService.reanalyze(request); reanalyzeService.reanalyze(request);
@ -636,8 +636,8 @@ public class RedactionIntegrationTest {
AnalyzeResult result = reanalyzeService.analyze(request); AnalyzeResult result = reanalyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var text = redactionStorageService.getText(TEST_PROJECT_ID, TEST_FILE_ID); var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getRedactionLogEntry().forEach(entry -> { redactionLog.getRedactionLogEntry().forEach(entry -> {
if (entry.isImage()) { if (entry.isImage()) {
@ -650,7 +650,7 @@ public class RedactionIntegrationTest {
System.out.println("first analysis duration: " + (end - start)); System.out.println("first analysis duration: " + (end - start));
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) { try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) {
fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_PROJECT_ID, TEST_FILE_ID))); fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID)));
} }
int correctFound = 0; int correctFound = 0;
@ -683,9 +683,9 @@ public class RedactionIntegrationTest {
dictionary.get(VERTEBRATE).add("s-metolachlor"); dictionary.get(VERTEBRATE).add("s-metolachlor");
reanlysisVersions.put("s-metolachlor", 3L); reanlysisVersions.put("s-metolachlor", 3L);
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
start = System.currentTimeMillis(); start = System.currentTimeMillis();
AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request); AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request);
@ -694,7 +694,7 @@ public class RedactionIntegrationTest {
System.out.println("reanalysis analysis duration: " + (end - start)); System.out.println("reanalysis analysis duration: " + (end - start));
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.projectId(TEST_PROJECT_ID) .dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.build()); .build());
@ -715,7 +715,7 @@ public class RedactionIntegrationTest {
AnalyzeResult result = reanalyzeService.analyze(request); AnalyzeResult result = reanalyzeService.analyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.projectId(TEST_PROJECT_ID) .dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.build()); .build());
@ -786,7 +786,7 @@ public class RedactionIntegrationTest {
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.projectId(TEST_PROJECT_ID) .dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.build()); .build());
@ -811,9 +811,9 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
RedactionRequest redactionRequest = RedactionRequest.builder() RedactionRequest redactionRequest = RedactionRequest.builder()
.projectId(request.getProjectId()) .dossierId(request.getDossierId())
.fileId(request.getFileId()) .fileId(request.getFileId())
.ruleSetId(request.getRuleSetId()) .dossierTemplateId(request.getDossierTemplateId())
.build(); .build();
RedactionResult result = redactionController.classify(redactionRequest); RedactionResult result = redactionController.classify(redactionRequest);
@ -833,9 +833,9 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
RedactionRequest redactionRequest = RedactionRequest.builder() RedactionRequest redactionRequest = RedactionRequest.builder()
.projectId(request.getProjectId()) .dossierId(request.getDossierId())
.fileId(request.getFileId()) .fileId(request.getFileId())
.ruleSetId(request.getRuleSetId()) .dossierTemplateId(request.getDossierTemplateId())
.build(); .build();
RedactionResult result = redactionController.sections(redactionRequest); RedactionResult result = redactionController.sections(redactionRequest);
@ -855,9 +855,9 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
RedactionRequest redactionRequest = RedactionRequest.builder() RedactionRequest redactionRequest = RedactionRequest.builder()
.projectId(request.getProjectId()) .dossierId(request.getDossierId())
.fileId(request.getFileId()) .fileId(request.getFileId())
.ruleSetId(request.getRuleSetId()) .dossierTemplateId(request.getDossierTemplateId())
.build(); .build();
RedactionResult result = redactionController.htmlTables(redactionRequest); RedactionResult result = redactionController.htmlTables(redactionRequest);
@ -877,9 +877,9 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
RedactionRequest redactionRequest = RedactionRequest.builder() RedactionRequest redactionRequest = RedactionRequest.builder()
.projectId(request.getProjectId()) .dossierId(request.getDossierId())
.fileId(request.getFileId()) .fileId(request.getFileId())
.ruleSetId(request.getRuleSetId()) .dossierTemplateId(request.getDossierTemplateId())
.build(); .build();
RedactionResult result = redactionController.htmlTables(redactionRequest); RedactionResult result = redactionController.htmlTables(redactionRequest);
@ -899,7 +899,7 @@ public class RedactionIntegrationTest {
AnalyzeResult result = reanalyzeService.analyze(request); AnalyzeResult result = reanalyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getRedactionLogEntry().forEach(entry -> { redactionLog.getRedactionLogEntry().forEach(entry -> {
if (!entry.isHint()) { if (!entry.isHint()) {
@ -920,15 +920,15 @@ public class RedactionIntegrationTest {
private AnalyzeRequest prepareStorage(InputStream stream) { private AnalyzeRequest prepareStorage(InputStream stream) {
AnalyzeRequest request = AnalyzeRequest.builder() AnalyzeRequest request = AnalyzeRequest.builder()
.ruleSetId(TEST_RULESET_ID) .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.projectId(TEST_PROJECT_ID) .dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.lastProcessed(OffsetDateTime.now()) .lastProcessed(OffsetDateTime.now())
.build(); .build();
var bytes = IOUtils.toByteArray(stream); var bytes = IOUtils.toByteArray(stream);
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_PROJECT_ID, TEST_FILE_ID, FileType.ORIGIN), bytes); storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
return request; return request;
@ -947,7 +947,7 @@ public class RedactionIntegrationTest {
AnalyzeResult result = reanalyzeService.analyze(request); AnalyzeResult result = reanalyzeService.analyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.projectId(TEST_PROJECT_ID) .dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.build()); .build());

128
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java
View File

@ -82,7 +82,7 @@ public class EntityRedactionServiceTest {
@MockBean @MockBean
private LegalBasisClient legalBasisClient; private LegalBasisClient legalBasisClient;
private final static String TEST_RULESET_ID = "123"; private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
@Configuration @Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@ -138,20 +138,20 @@ public class EntityRedactionServiceTest {
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))) .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H.")))
.build(); .build();
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))) .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
} }
@ -165,19 +165,19 @@ public class EntityRedactionServiceTest {
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))) .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H.")))
.build(); .build();
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))) .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
} }
@ -188,21 +188,21 @@ public class EntityRedactionServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" + ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" +
" Supplement - Identity of the active substance - Reference list.pdf"); " Supplement - Identity of the active substance - Reference list.pdf");
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))) .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))) .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities() assertThat(classifiedDoc.getEntities()
.entrySet() .entrySet()
.stream() .stream()
@ -210,7 +210,7 @@ public class EntityRedactionServiceTest {
pdfFileResource = new ClassPathResource("files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of " + pdfFileResource = new ClassPathResource("files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of " +
"the plant protection product.pdf"); "the plant protection product.pdf");
classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities() assertThat(classifiedDoc.getEntities()
.entrySet() .entrySet()
.stream() .stream()
@ -221,21 +221,21 @@ public class EntityRedactionServiceTest {
public void testFalsePositiveInWrongCell() throws IOException { public void testFalsePositiveInWrongCell() throws IOException {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))) .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))) .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))) .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))))
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream() assertThat(classifiedDoc.getEntities().get(1).stream()
.filter(entity -> entity.getMatchedRule() == 9) .filter(entity -> entity.getMatchedRule() == 9)
@ -283,26 +283,26 @@ public class EntityRedactionServiceTest {
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + "\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
" section.redactBetween(\"Contact:\", \"Tel.:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " section.redactBetween(\"Contact:\", \"Tel.:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
" end"; " end";
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet()); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(RULES_VERSION.incrementAndGet());
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules)); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new RulesResponse(tableRules));
droolsExecutionService.updateRules(TEST_RULESET_ID); droolsExecutionService.updateRules(TEST_DOSSIER_TEMPLATE_ID);
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))) .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))) .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream() assertThat(classifiedDoc.getEntities().get(1).stream()
.filter(entity -> entity.getMatchedRule() == 6) .filter(entity -> entity.getMatchedRule() == 6)
@ -322,26 +322,26 @@ public class EntityRedactionServiceTest {
"Section(searchText.toLowerCase().contains(\"batches produced at\"))\n" + " then\n" + " section" + "Section(searchText.toLowerCase().contains(\"batches produced at\"))\n" + " then\n" + " section" +
".redactIfPrecededBy(\"batches produced at\", \"sponsor\", 11, \"Redacted because it represents a " + ".redactIfPrecededBy(\"batches produced at\", \"sponsor\", 11, \"Redacted because it represents a " +
"sponsor company\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " end"; "sponsor company\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " end";
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet()); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(RULES_VERSION.incrementAndGet());
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules)); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new RulesResponse(tableRules));
droolsExecutionService.updateRules(TEST_RULESET_ID); droolsExecutionService.updateRules(TEST_DOSSIER_TEMPLATE_ID);
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf");
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse authorResponse = DictionaryResponse.builder() DictionaryResponse authorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(authorResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(authorResponse);
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))) .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))))
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream() assertThat(classifiedDoc.getEntities().get(1).stream()
.filter(entity -> entity.getMatchedRule() == 11) .filter(entity -> entity.getMatchedRule() == 11)
@ -359,19 +359,19 @@ public class EntityRedactionServiceTest {
.entries(toDictionaryEntry(Arrays.asList("Bissig R.", "Thanei P."))) .entries(toDictionaryEntry(Arrays.asList("Bissig R.", "Thanei P.")))
.build(); .build();
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))) .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8);
assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y
@ -382,15 +382,15 @@ public class EntityRedactionServiceTest {
.entries(toDictionaryEntry(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C."))) .entries(toDictionaryEntry(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C.")))
.build(); .build();
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
addressResponse = DictionaryResponse.builder() addressResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))) .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3);
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9);
@ -407,19 +407,19 @@ public class EntityRedactionServiceTest {
.entries(toDictionaryEntry(Collections.singletonList("Aldershof S."))) .entries(toDictionaryEntry(Collections.singletonList("Aldershof S.")))
.build(); .build();
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))) .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId"); entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId");
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6);
} }
@ -446,31 +446,31 @@ public class EntityRedactionServiceTest {
"\"Yes\"))\n" + "\"Yes\"))\n" +
" then\n" + " then\n" +
" section.redactCell(\"Author(s)\", 9, \"name\", false, \"Redacted because row is a vertebrate study\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " section.redactCell(\"Author(s)\", 9, \"name\", false, \"Redacted because row is a vertebrate study\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
" section.redact(\"address\", 9, \"Redacted because row is a vertebrate study\", \"Reg (EC) No" + " section.redact(\"address\", 9, \"Redacted because row is a vertebrate sgitudy\", \"Reg (EC) No" +
" 1107/2009 Art. 63 (2g)\");\n" + " 1107/2009 Art. 63 (2g)\");\n" +
" section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" + " section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" +
" end"; " end";
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet()); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(RULES_VERSION.incrementAndGet());
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules)); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new RulesResponse(tableRules));
TypeResponse typeResponse = TypeResponse.builder() TypeResponse typeResponse = TypeResponse.builder()
.types(Arrays.asList( .types(Arrays.asList(
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(AUTHOR_CODE).hexColor("#ffff00").build(), TypeResult.builder().ruleSetId(TEST_DOSSIER_TEMPLATE_ID).type(AUTHOR_CODE).hexColor("#ffff00").build(),
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(ADDRESS_CODE).hexColor("#ff00ff").build(), TypeResult.builder().ruleSetId(TEST_DOSSIER_TEMPLATE_ID).type(ADDRESS_CODE).hexColor("#ff00ff").build(),
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(SPONSOR_CODE).hexColor("#00ffff").build())) TypeResult.builder().ruleSetId(TEST_DOSSIER_TEMPLATE_ID).type(SPONSOR_CODE).hexColor("#00ffff").build()))
.build(); .build();
when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getAllTypes(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(typeResponse); when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(typeResponse);
// Default empty return to prevent NPEs // Default empty return to prevent NPEs
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
Colors colors = new Colors(); Colors colors = new Colors();
colors.setDefaultColor("#acfc00"); colors.setDefaultColor("#acfc00");
@ -478,7 +478,7 @@ public class EntityRedactionServiceTest {
colors.setRequestAdd("#04b093"); colors.setRequestAdd("#04b093");
colors.setRequestRemove("#04b093"); colors.setRequestRemove("#04b093");
when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
} }