Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RuleSetId integration and drools update

Browse Source
This commit is contained in:
Timo 2021-01-06 01:42:14 +02:00
parent c2fd749fec
commit 5aba4b69ba
12 changed files with 410 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/pom.xml
View File

@ -32,7 +32,7 @@
<dependency> <dependency>
<groupId>com.iqser.red</groupId> <groupId>com.iqser.red</groupId>
<artifactId>platform-commons-dependency</artifactId> <artifactId>platform-commons-dependency</artifactId>
<version>1.1.0</version> <version>1.1.7</version>
<scope>import</scope> <scope>import</scope>
<type>pom</type> <type>pom</type>
</dependency> </dependency>

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java
View File

@ -12,6 +12,7 @@ import lombok.NoArgsConstructor;
public class RedactionRequest { public class RedactionRequest {
private byte[] document; private byte[] document;
private String ruleSetId;
private boolean flatRedaction; private boolean flatRedaction;
private ManualRedactions manualRedactions; private ManualRedactions manualRedactions;
} }

2
redaction-service-v1/redaction-service-server-v1/pom.xml
View File

@ -20,7 +20,7 @@
<dependency> <dependency>
<groupId>com.iqser.red.service</groupId> <groupId>com.iqser.red.service</groupId>
<artifactId>configuration-service-api-v1</artifactId> <artifactId>configuration-service-api-v1</artifactId>
<version>1.3.7</version> <version>2.0.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.drools</groupId> <groupId>org.drools</groupId>

51
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java
View File

@ -1,16 +1,8 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import java.io.ByteArrayInputStream; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
import java.io.InputStream; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import java.nio.charset.StandardCharsets; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration; import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
@ -18,54 +10,17 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Import;
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@Import({DefaultWebMvcConfiguration.class}) @Import({DefaultWebMvcConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class) @EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class) @EnableConfigurationProperties(RedactionServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class}) @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
public class Application { public class Application {
@Autowired
private RulesClient rulesClient;
public static void main(String[] args) { public static void main(String[] args) {
SpringApplication.run(Application.class, args); SpringApplication.run(Application.class, args);
} }
@Bean
public KieContainer kieContainer() {
try {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
RulesResponse rules = rulesClient.getRules();
if (StringUtils.isEmpty(rules.getRules())) {
throw new RuntimeException("Rules cannot be empty.");
}
InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
.newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();
return kieServices.newKieContainer(kieModule.getReleaseId());
} catch (Exception e) {
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
}
}
} }

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -1,14 +1,5 @@
package com.iqser.red.service.redaction.v1.server.controller; package com.iqser.red.service.redaction.v1.server.controller;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionRequest; import com.iqser.red.service.redaction.v1.model.RedactionRequest;
@ -25,9 +16,16 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService; import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService;
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService; import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
@Slf4j @Slf4j
@RestController @RestController
@ -51,9 +49,9 @@ public class RedactionController implements RedactionResource {
log.info("Document structure analysis successful, starting redaction analysis..."); log.info("Document structure analysis successful, starting redaction analysis...");
entityRedactionService.processDocument(classifiedDoc, redactionRequest.getManualRedactions()); entityRedactionService.processDocument(classifiedDoc, redactionRequest.getRuleSetId(), redactionRequest.getManualRedactions());
annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest
.getManualRedactions()); .getManualRedactions(), redactionRequest.getRuleSetId());
log.info("Redaction analysis successful..."); log.info("Redaction analysis successful...");
@ -152,7 +150,7 @@ public class RedactionController implements RedactionResource {
return RedactionResult.builder() return RedactionResult.builder()
.document(byteArrayOutputStream.toByteArray()) .document(byteArrayOutputStream.toByteArray())
.numberOfPages(numberOfPages) .numberOfPages(numberOfPages)
.redactionLog(new RedactionLog(redactionLogEntities,dictionaryVersion, rulesVersion)) .redactionLog(new RedactionLog(redactionLogEntities, dictionaryVersion, rulesVersion))
.sectionGrid(sectionGrid) .sectionGrid(sectionGrid)
.build(); .build();
} }

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java Normal file
View File

@ -0,0 +1,24 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import lombok.Data;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Data
public class DictionaryRepresentation {
private String ruleSetId;
private long dictionaryVersion = -1;
private List<DictionaryModel> dictionary = new ArrayList<>();
private float[] defaultColor;
private float[] requestAddColor;
private float[] requestRemoveColor;
private float[] notRedactedColor;
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
}

136
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -1,5 +1,19 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.configuration.v1.api.model.Colors;
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.SerializationUtils;
import org.springframework.stereotype.Service;
import java.awt.Color; import java.awt.Color;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
@ -10,22 +24,6 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.SerializationUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.configuration.v1.api.model.Colors;
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import feign.FeignException;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@ -34,60 +32,51 @@ public class DictionaryService {
private final DictionaryClient dictionaryClient; private final DictionaryClient dictionaryClient;
@Getter
private long dictionaryVersion = -1;
@Getter private Map<String, DictionaryRepresentation> dictionariesByRuleSets = new HashMap<>();
private List<DictionaryModel> dictionary = new ArrayList<>();
@Getter
private float[] defaultColor;
@Getter
private float[] requestAddColor;
@Getter
private float[] requestRemoveColor;
@Getter
private float[] notRedactedColor;
private Map<String, DictionaryModel> localAccessMap = new HashMap<>(); public void updateDictionary(String ruleSetId) {
long version = dictionaryClient.getVersion(ruleSetId);
var foundDictionary = dictionariesByRuleSets.get(ruleSetId);
public void updateDictionary() { if (foundDictionary == null || version > foundDictionary.getDictionaryVersion()) {
updateDictionaryEntry(ruleSetId, version);
long version = dictionaryClient.getVersion();
if (version > dictionaryVersion) {
dictionaryVersion = version;
updateDictionaryEntry();
} }
} }
private void updateDictionaryEntry() { private void updateDictionaryEntry(String ruleSetId, long version) {
try { try {
TypeResponse typeResponse = dictionaryClient.getAllTypes(); DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
TypeResponse typeResponse = dictionaryClient.getAllTypes(ruleSetId);
if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) { if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
dictionary = typeResponse.getTypes() List<DictionaryModel> dictionary = typeResponse.getTypes()
.stream() .stream()
.map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t .map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
.isHint(), t.isRecommendation(), convertEntries(t), new HashSet<>())) .isHint(), t.isRecommendation(), convertEntries(t), new HashSet<>()))
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.collect(Collectors.toList()); .collect(Collectors.toList());
localAccessMap.clear();
dictionary.forEach(dm -> localAccessMap.put(dm.getType(), dm));
Colors colors = dictionaryClient.getColors(); dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
defaultColor = convertColor(colors.getDefaultColor());
requestAddColor = convertColor(colors.getRequestAdd()); Colors colors = dictionaryClient.getColors(ruleSetId);
requestRemoveColor = convertColor(colors.getRequestRemove());
notRedactedColor = convertColor(colors.getNotRedacted()); dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
dictionaryRepresentation.setRuleSetId(ruleSetId);
dictionaryRepresentation.setDictionaryVersion(version);
dictionaryRepresentation.setDictionary(dictionary);
dictionariesByRuleSets.put(ruleSetId, dictionaryRepresentation);
} }
} catch (FeignException e) { } catch (FeignException e) {
log.warn("Got some unknown feignException", e); log.warn("Got some unknown feignException", e);
@ -96,12 +85,12 @@ public class DictionaryService {
} }
public void updateExternalDictionary(Dictionary dictionary){ public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) {
dictionary.getDictionaryModels().forEach(dm -> { dictionary.getDictionaryModels().forEach(dm -> {
if(dm.isRecommendation() && !dm.getLocalEntries().isEmpty()){ if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
dictionaryClient.addEntries(dm.getType(), new ArrayList<>(dm.getLocalEntries()), false); dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false);
long externalVersion = dictionaryClient.getVersion(); long externalVersion = dictionaryClient.getVersion(ruleSetId);
if(externalVersion == dictionary.getVersion() + 1){ if (externalVersion == dictionary.getVersion() + 1) {
dictionary.setVersion(externalVersion); dictionary.setVersion(externalVersion);
} }
} }
@ -112,13 +101,13 @@ public class DictionaryService {
private Set<String> convertEntries(TypeResult t) { private Set<String> convertEntries(TypeResult t) {
if (t.isCaseInsensitive()) { if (t.isCaseInsensitive()) {
return dictionaryClient.getDictionaryForType(t.getType()) return dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
.getEntries() .getEntries()
.stream() .stream()
.map(String::toLowerCase) .map(String::toLowerCase)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
} else { } else {
return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType()).getEntries()); return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId()).getEntries());
} }
} }
@ -130,9 +119,9 @@ public class DictionaryService {
} }
public boolean isCaseInsensitiveDictionary(String type) { public boolean isCaseInsensitiveDictionary(String type, String ruleSetId) {
DictionaryModel dictionaryModel = localAccessMap.get(type); DictionaryModel dictionaryModel = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
if (dictionaryModel != null) { if (dictionaryModel != null) {
return dictionaryModel.isCaseInsensitive(); return dictionaryModel.isCaseInsensitive();
} }
@ -140,28 +129,28 @@ public class DictionaryService {
} }
public float[] getColor(String type) { public float[] getColor(String type, String ruleSetId) {
DictionaryModel model = localAccessMap.get(type); DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
if (model != null) { if (model != null) {
return model.getColor(); return model.getColor();
} }
return defaultColor; return dictionariesByRuleSets.get(ruleSetId).getDefaultColor();
} }
public boolean isHint(String type) { public boolean isHint(String type, String ruleSetId) {
DictionaryModel model = localAccessMap.get(type); DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
if (model != null) { if (model != null) {
return model.isHint(); return model.isHint();
} }
return false; return false;
} }
public boolean isRecommendation(String type) { public boolean isRecommendation(String type, String ruleSetId) {
DictionaryModel model = localAccessMap.get(type); DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
if (model != null) { if (model != null) {
return model.isRecommendation(); return model.isRecommendation();
} }
@ -169,14 +158,27 @@ public class DictionaryService {
} }
public Dictionary getDeepCopyDictionary(){ public Dictionary getDeepCopyDictionary(String ruleSetId) {
List<DictionaryModel> copy = new ArrayList<>(); List<DictionaryModel> copy = new ArrayList<>();
var representation = dictionariesByRuleSets.get(ruleSetId);
var dictionary = dictionariesByRuleSets.get(ruleSetId).getDictionary();
dictionary.forEach(dm -> { dictionary.forEach(dm -> {
copy.add(SerializationUtils.clone(dm)); copy.add(SerializationUtils.clone(dm));
}); });
return new Dictionary(copy, dictionaryVersion); return new Dictionary(copy, representation.getDictionaryVersion());
} }
public float[] getRequestRemoveColor(String ruleSetId) {
return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
}
public float[] getNotRedactedColor(String ruleSetId) {
return dictionariesByRuleSets.get(ruleSetId).getNotRedactedColor();
}
public float[] getRequestAddColor(String ruleSetId) {
return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
}
} }

80
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -1,9 +1,11 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.io.ByteArrayInputStream; import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
import java.io.InputStream; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import java.nio.charset.StandardCharsets; import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.kie.api.KieServices; import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder; import org.kie.api.builder.KieBuilder;
@ -11,15 +13,13 @@ import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule; import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieContainer;
import org.kie.api.runtime.KieSession; import org.kie.api.runtime.KieSession;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import java.io.ByteArrayInputStream;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; import java.io.InputStream;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import lombok.Getter; import java.util.Map;
import lombok.RequiredArgsConstructor;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@ -27,13 +27,22 @@ public class DroolsExecutionService {
private final RulesClient rulesClient; private final RulesClient rulesClient;
@Autowired private Map<String, KieContainer> kieContainers = new HashMap<>();
private KieContainer kieContainer;
public KieContainer getKieContainer(String ruleSetId) {
KieContainer container = kieContainers.get(ruleSetId);
if (container == null) {
return createOrUpdateKieContainer(ruleSetId);
} else {
return container;
}
}
@Getter @Getter
private long rulesVersion = -1; private long rulesVersion = -1;
public Section executeRules(Section section) { public Section executeRules(KieContainer kieContainer, Section section) {
KieSession kieSession = kieContainer.newKieSession(); KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("section", section); kieSession.setGlobal("section", section);
@ -45,34 +54,53 @@ public class DroolsExecutionService {
} }
public void updateRules() { public KieContainer updateRules(String ruleSetId) {
long version = rulesClient.getVersion(); long version = rulesClient.getVersion(ruleSetId);
if (version > rulesVersion) { if (version > rulesVersion) {
rulesVersion = version; rulesVersion = version;
updateRules(rulesClient.getRules().getRules()); return createOrUpdateKieContainer(ruleSetId);
} }
return getKieContainer(ruleSetId);
} }
public void updateRules(String drlAsString) { private KieContainer createOrUpdateKieContainer(String ruleSetId) {
try { try {
if (StringUtils.isEmpty(drlAsString)) {
RulesResponse rules = rulesClient.getRules(ruleSetId);
if (rules == null || StringUtils.isEmpty(rules.getRules())) {
throw new RuntimeException("Rules cannot be empty."); throw new RuntimeException("Rules cannot be empty.");
} }
KieServices kieServices = KieServices.Factory.get(); KieServices kieServices = KieServices.Factory.get();
InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8)); KieModule kieModule = getKieModule(ruleSetId, rules.getRules(), kieServices);
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input)); var container = kieContainers.get(ruleSetId);
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); if (container != null) {
kieBuilder.buildAll(); container.updateToVersion(kieModule.getReleaseId());
KieModule kieModule = kieBuilder.getKieModule(); return container;
kieContainer.updateToVersion(kieModule.getReleaseId()); }
container = kieServices.newKieContainer(kieModule.getReleaseId());
kieContainers.put(ruleSetId, container);
return container;
} catch (Exception e) { } catch (Exception e) {
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e); throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
} }
} }
private KieModule getKieModule(String ruleSetId, String rules, KieServices kieServices) {
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/main/resources/drools/rules" + ruleSetId + ".drl", kieServices.getResources()
.newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
return kieBuilder.getKieModule();
}
} }

49
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -1,20 +1,5 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactions; import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
@ -32,9 +17,23 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchab
import com.iqser.red.service.redaction.v1.server.redaction.utils.PositionUtil; import com.iqser.red.service.redaction.v1.server.redaction.utils.PositionUtil;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j @Slf4j
@Service @Service
@ -46,15 +45,15 @@ public class EntityRedactionService {
private final SurroundingWordsService surroundingWordsService; private final SurroundingWordsService surroundingWordsService;
public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) { public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
dictionaryService.updateDictionary(); dictionaryService.updateDictionary(ruleSetId);
droolsExecutionService.updateRules(); KieContainer container = droolsExecutionService.updateRules(ruleSetId);
long rulesVersion = droolsExecutionService.getRulesVersion(); long rulesVersion = droolsExecutionService.getRulesVersion();
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, manualRedactions, dictionary, false, null)); Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
if (dictionary.hasLocalEntries()) { if (dictionary.hasLocalEntries()) {
@ -66,7 +65,7 @@ public class EntityRedactionService {
} }
}); });
Set<Entity> foundByLocal = findEntities(classifiedDoc, manualRedactions, dictionary, true, hintsPerSectionNumber); Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
// HashSet keeps the older value, but we want the new only. // HashSet keeps the older value, but we want the new only.
documentEntities.removeAll(foundByLocal); documentEntities.removeAll(foundByLocal);
documentEntities.addAll(foundByLocal); documentEntities.addAll(foundByLocal);
@ -90,14 +89,14 @@ public class EntityRedactionService {
} }
} }
dictionaryService.updateExternalDictionary(dictionary); dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
classifiedDoc.setDictionaryVersion(dictionary.getVersion()); classifiedDoc.setDictionaryVersion(dictionary.getVersion());
classifiedDoc.setRulesVersion(rulesVersion); classifiedDoc.setRulesVersion(rulesVersion);
} }
private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions, Dictionary dictionary, private Set<Entity> findEntities(Document classifiedDoc, KieContainer kieContainer, ManualRedactions manualRedactions, Dictionary dictionary,
boolean local, Map<Integer, Set<Entity>> hintsPerSectionNumber) { boolean local, Map<Integer, Set<Entity>> hintsPerSectionNumber) {
Set<Entity> documentEntities = new HashSet<>(); Set<Entity> documentEntities = new HashSet<>();
@ -120,7 +119,7 @@ public class EntityRedactionService {
} }
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
Section analysedRowSection = droolsExecutionService.executeRules(sectionSearchableTextPair.getSection()); Section analysedRowSection = droolsExecutionService.executeRules(kieContainer,sectionSearchableTextPair.getSection());
documentEntities.addAll(analysedRowSection.getEntities()); documentEntities.addAll(analysedRowSection.getEntities());
analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> { analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {

112
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java
View File

@ -1,28 +1,5 @@
package com.iqser.red.service.redaction.v1.server.visualization.service; package com.iqser.red.service.redaction.v1.server.visualization.service;
import java.awt.Color;
import java.io.IOException;
import java.util.ArrayList;
import java.util.GregorianCalendar;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
import org.apache.pdfbox.text.TextPosition;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.CellRectangle; import com.iqser.red.service.redaction.v1.model.CellRectangle;
import com.iqser.red.service.redaction.v1.model.Comment; import com.iqser.red.service.redaction.v1.model.Comment;
import com.iqser.red.service.redaction.v1.model.IdRemoval; import com.iqser.red.service.redaction.v1.model.IdRemoval;
@ -44,8 +21,29 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.DictionarySer
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
import org.apache.pdfbox.text.TextPosition;
import org.springframework.stereotype.Service;
import java.awt.Color;
import java.io.IOException;
import java.util.ArrayList;
import java.util.GregorianCalendar;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@ -55,7 +53,7 @@ public class AnnotationHighlightService {
public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction, public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction,
ManualRedactions manualRedactions) throws IOException { ManualRedactions manualRedactions, String ruleSetId) throws IOException {
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions); Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
@ -66,11 +64,11 @@ public class AnnotationHighlightService {
drawSectionFrames(document, classifiedDoc, flatRedaction, pdPage, page); drawSectionFrames(document, classifiedDoc, flatRedaction, pdPage, page);
if (classifiedDoc.getEntities().get(page) != null) { if (classifiedDoc.getEntities().get(page) != null) {
addAnnotations(pdPage, classifiedDoc, flatRedaction, manualRedactions, page); addAnnotations(pdPage, classifiedDoc, flatRedaction, manualRedactions, page, ruleSetId);
} }
if (manualRedactionPages.contains(page)) { if (manualRedactionPages.contains(page)) {
addManualAnnotations(pdPage, classifiedDoc, manualRedactions, page); addManualAnnotations(pdPage, classifiedDoc, manualRedactions, page, ruleSetId);
} }
} }
} }
@ -94,7 +92,7 @@ public class AnnotationHighlightService {
private void addAnnotations(PDPage pdPage, Document classifiedDoc, boolean flatRedaction, private void addAnnotations(PDPage pdPage, Document classifiedDoc, boolean flatRedaction,
ManualRedactions manualRedactions, int page) throws IOException { ManualRedactions manualRedactions, int page, String ruleSetId) throws IOException {
List<PDAnnotation> annotations = pdPage.getAnnotations(); List<PDAnnotation> annotations = pdPage.getAnnotations();
@ -104,7 +102,7 @@ public class AnnotationHighlightService {
entityLoop: entityLoop:
for (Entity entity : classifiedDoc.getEntities().get(page)) { for (Entity entity : classifiedDoc.getEntities().get(page)) {
if (flatRedaction && !isRedactionType(entity)) { if (flatRedaction && !isRedactionType(entity, ruleSetId)) {
continue; continue;
} }
@ -113,7 +111,7 @@ public class AnnotationHighlightService {
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity); RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, ruleSetId);
if (processedIds.contains(entityPositionSequence.getId())) { if (processedIds.contains(entityPositionSequence.getId())) {
// TODO refactor this outer loop jump as soon as we have the time. // TODO refactor this outer loop jump as soon as we have the time.
@ -161,7 +159,7 @@ public class AnnotationHighlightService {
redactionLogEntry.getPositions().addAll(rectanglesPerLine); redactionLogEntry.getPositions().addAll(rectanglesPerLine);
annotations.addAll(createAnnotation(rectanglesPerLine, entityPositionSequence.getId(), createAnnotationContent(entity), getColor(entity, requestedToRemove), comments, !isHint(entity))); annotations.addAll(createAnnotation(rectanglesPerLine, entityPositionSequence.getId(), createAnnotationContent(entity), getColor(entity, ruleSetId, requestedToRemove), comments, !isHint(entity, ruleSetId)));
} }
redactionLogEntry.setId(entityPositionSequence.getId()); redactionLogEntry.setId(entityPositionSequence.getId());
@ -203,7 +201,7 @@ public class AnnotationHighlightService {
private void addManualAnnotations(PDPage pdPage, Document classifiedDoc, ManualRedactions manualRedactions, private void addManualAnnotations(PDPage pdPage, Document classifiedDoc, ManualRedactions manualRedactions,
int page) throws IOException { int page, String ruleSetId) throws IOException {
if (manualRedactions == null) { if (manualRedactions == null) {
return; return;
@ -215,7 +213,7 @@ public class AnnotationHighlightService {
String id = manualRedactionEntry.getId(); String id = manualRedactionEntry.getId();
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id); RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, ruleSetId);
List<Rectangle> rectanglesOnPage = new ArrayList<>(); List<Rectangle> rectanglesOnPage = new ArrayList<>();
for (Rectangle rectangle : manualRedactionEntry.getPositions()) { for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
@ -227,7 +225,7 @@ public class AnnotationHighlightService {
if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) { if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) {
annotations.addAll(createAnnotation(rectanglesOnPage, id, createAnnotationContent(manualRedactionEntry), getColorForManualAdd(manualRedactionEntry annotations.addAll(createAnnotation(rectanglesOnPage, id, createAnnotationContent(manualRedactionEntry), getColorForManualAdd(manualRedactionEntry
.getType(), manualRedactionEntry.getStatus()), manualRedactions.getComments().get(id), true)); .getType(), ruleSetId, manualRedactionEntry.getStatus()), manualRedactions.getComments().get(id), true));
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry); classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
} }
} }
@ -240,11 +238,11 @@ public class AnnotationHighlightService {
} }
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id) { private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, String ruleSetId) {
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.id(id) .id(id)
.color(getColor(manualRedactionEntry.getType())) .color(getColor(manualRedactionEntry.getType(), ruleSetId))
.reason(manualRedactionEntry.getReason()) .reason(manualRedactionEntry.getReason())
.legalBasis(manualRedactionEntry.getLegalBasis()) .legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue()) .value(manualRedactionEntry.getValue())
@ -261,17 +259,17 @@ public class AnnotationHighlightService {
} }
private RedactionLogEntry createRedactionLogEntry(Entity entity) { private RedactionLogEntry createRedactionLogEntry(Entity entity, String ruleSetId) {
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.color(getColor(entity, false)) .color(getColor(entity, ruleSetId, false))
.reason(entity.getRedactionReason()) .reason(entity.getRedactionReason())
.legalBasis(entity.getLegalBasis()) .legalBasis(entity.getLegalBasis())
.value(entity.getWord()) .value(entity.getWord())
.type(entity.getType()) .type(entity.getType())
.redacted(entity.isRedaction()) .redacted(entity.isRedaction())
.isHint(isHint(entity)) .isHint(isHint(entity, ruleSetId))
.isRecommendation(isRecommendation(entity)) .isRecommendation(isRecommendation(entity, ruleSetId))
.section(entity.getHeadline()) .section(entity.getHeadline())
.sectionNumber(entity.getSectionNumber()) .sectionNumber(entity.getSectionNumber())
.matchedRule(entity.getMatchedRule()) .matchedRule(entity.getMatchedRule())
@ -389,49 +387,49 @@ public class AnnotationHighlightService {
} }
private boolean isRedactionType(Entity entity) { private boolean isRedactionType(Entity entity, String ruleSetId) {
if (!entity.isRedaction()) { if (!entity.isRedaction()) {
return false; return false;
} }
return !isHint(entity); return !isHint(entity, ruleSetId);
} }
private float[] getColor(Entity entity, boolean requestedToRemove) { private float[] getColor(Entity entity, String ruleSetId, boolean requestedToRemove) {
if (requestedToRemove) { if (requestedToRemove) {
return dictionaryService.getRequestRemoveColor(); return dictionaryService.getRequestRemoveColor(ruleSetId);
} }
if (!entity.isRedaction() && !isHint(entity)) { if (!entity.isRedaction() && !isHint(entity, ruleSetId)) {
return dictionaryService.getNotRedactedColor(); return dictionaryService.getNotRedactedColor(ruleSetId);
} }
return dictionaryService.getColor(entity.getType()); return dictionaryService.getColor(entity.getType(), ruleSetId);
} }
private float[] getColorForManualAdd(String type, Status status) { private float[] getColorForManualAdd(String type, String ruleSetId, Status status) {
if (status.equals(Status.REQUESTED)) { if (status.equals(Status.REQUESTED)) {
return dictionaryService.getRequestAddColor(); return dictionaryService.getRequestAddColor(ruleSetId);
} else if (status.equals(Status.DECLINED)) { } else if (status.equals(Status.DECLINED)) {
return dictionaryService.getNotRedactedColor(); return dictionaryService.getNotRedactedColor(ruleSetId);
} }
return getColor(type); return getColor(type, ruleSetId);
} }
private float[] getColor(String type) { private float[] getColor(String type, String ruleSetId) {
return dictionaryService.getColor(type); return dictionaryService.getColor(type, ruleSetId);
} }
private boolean isHint(Entity entity) { private boolean isHint(Entity entity, String ruleSetId) {
return dictionaryService.isHint(entity.getType()); return dictionaryService.isHint(entity.getType(), ruleSetId);
} }
private boolean isRecommendation(Entity entity) { private boolean isRecommendation(Entity entity, String ruleSetId) {
return dictionaryService.isRecommendation(entity.getType()); return dictionaryService.isRecommendation(entity.getType(), ruleSetId);
} }

128
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,45 +1,5 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import com.iqser.red.service.configuration.v1.api.model.Colors; import com.iqser.red.service.configuration.v1.api.model.Colors;
import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse; import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
import com.iqser.red.service.configuration.v1.api.model.RulesResponse; import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
@ -60,6 +20,45 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT;
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = DEFINED_PORT) @SpringBootTest(webEnvironment = DEFINED_PORT)
@ -100,6 +99,8 @@ public class RedactionIntegrationTest {
private final Map<String, Boolean> recommendationTypeMap = new HashMap<>(); private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
private final Colors colors = new Colors(); private final Colors colors = new Colors();
private final static String TEST_RULESET_ID = "123";
@TestConfiguration @TestConfiguration
public static class RedactionIntegrationTestConfiguration { public static class RedactionIntegrationTestConfiguration {
@ -125,28 +126,28 @@ public class RedactionIntegrationTest {
@Before @Before
public void stubClients() { public void stubClients() {
when(rulesClient.getVersion()).thenReturn(0L); when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(0L);
when(rulesClient.getRules()).thenReturn(new RulesResponse(RULES)); when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(RULES));
loadDictionaryForTest(); loadDictionaryForTest();
loadTypeForTest(); loadTypeForTest();
when(dictionaryClient.getVersion()).thenReturn(0L); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypes()).thenReturn(TypeResponse.builder().types(getTypeResponse()).build()); when(dictionaryClient.getAllTypes(TEST_RULESET_ID)).thenReturn(TypeResponse.builder().types(getTypeResponse()).build());
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE)); when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));
when(dictionaryClient.getDictionaryForType(ADDRESS)).thenReturn(getDictionaryResponse(ADDRESS)); when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(ADDRESS));
when(dictionaryClient.getDictionaryForType(AUTHOR)).thenReturn(getDictionaryResponse(AUTHOR)); when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(AUTHOR));
when(dictionaryClient.getDictionaryForType(SPONSOR)).thenReturn(getDictionaryResponse(SPONSOR)); when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(SPONSOR));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR)); when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR)); when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY)); when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(HINT_ONLY));
when(dictionaryClient.getDictionaryForType(MUST_REDACT)).thenReturn(getDictionaryResponse(MUST_REDACT)); when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(MUST_REDACT));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION)); when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION));
when(dictionaryClient.getDictionaryForType(TEST_METHOD)).thenReturn(getDictionaryResponse(TEST_METHOD)); when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(TEST_METHOD));
when(dictionaryClient.getDictionaryForType(PII)).thenReturn(getDictionaryResponse(PII)); when(dictionaryClient.getDictionaryForType(PII, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PII));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR)); when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS)); when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE)).thenReturn(getDictionaryResponse(FALSE_POSITIVE)); when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
when(dictionaryClient.getColors()).thenReturn(colors); when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
} }
@ -306,6 +307,7 @@ public class RedactionIntegrationTest {
.stream() .stream()
.map(typeColor -> TypeResult.builder() .map(typeColor -> TypeResult.builder()
.type(typeColor.getKey()) .type(typeColor.getKey())
.ruleSetId(TEST_RULESET_ID)
.hexColor(typeColor.getValue()) .hexColor(typeColor.getValue())
.isHint(hintTypeMap.get(typeColor.getKey())) .isHint(hintTypeMap.get(typeColor.getKey()))
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
@ -342,6 +344,7 @@ public class RedactionIntegrationTest {
} }
for (File path : input) { for (File path : input) {
RedactionRequest request = RedactionRequest.builder() RedactionRequest request = RedactionRequest.builder()
.ruleSetId(TEST_RULESET_ID)
.document(IOUtils.toByteArray(new FileInputStream(path))) .document(IOUtils.toByteArray(new FileInputStream(path)))
.build(); .build();
System.out.println("Redacting file : " + path.getName()); System.out.println("Redacting file : " + path.getName());
@ -390,6 +393,7 @@ public class RedactionIntegrationTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
RedactionRequest request = RedactionRequest.builder() RedactionRequest request = RedactionRequest.builder()
.ruleSetId(TEST_RULESET_ID)
.document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build(); .build();
request.setFlatRedaction(false); request.setFlatRedaction(false);
@ -420,6 +424,7 @@ public class RedactionIntegrationTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
RedactionRequest request = RedactionRequest.builder() RedactionRequest request = RedactionRequest.builder()
.ruleSetId(TEST_RULESET_ID)
.document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build(); .build();
@ -471,6 +476,7 @@ public class RedactionIntegrationTest {
manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
RedactionRequest request = RedactionRequest.builder() RedactionRequest request = RedactionRequest.builder()
.ruleSetId(TEST_RULESET_ID)
.document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.manualRedactions(manualRedactions) .manualRedactions(manualRedactions)
.build(); .build();
@ -565,6 +571,7 @@ public class RedactionIntegrationTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Phantom Cells.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Phantom Cells.pdf");
RedactionRequest request = RedactionRequest.builder() RedactionRequest request = RedactionRequest.builder()
.ruleSetId(TEST_RULESET_ID)
.document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build(); .build();
request.setFlatRedaction(false); request.setFlatRedaction(false);
@ -586,9 +593,10 @@ public class RedactionIntegrationTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf");
RedactionRequest request = RedactionRequest.builder() RedactionRequest request = RedactionRequest.builder()
.ruleSetId(TEST_RULESET_ID)
.flatRedaction(false)
.document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build(); .build();
request.setFlatRedaction(false);
RedactionResult result = redactionController.redact(request); RedactionResult result = redactionController.redact(request);

197
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java
View File

@ -1,25 +1,18 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import static org.assertj.core.api.Assertions.assertThat; import com.iqser.red.service.configuration.v1.api.model.Colors;
import static org.mockito.Mockito.when; import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
import java.io.BufferedReader; import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
import java.io.ByteArrayInputStream; import com.iqser.red.service.configuration.v1.api.model.TypeResult;
import java.io.IOException; import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import java.io.InputStream; import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import java.io.InputStreamReader; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import java.net.URL; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import java.nio.charset.StandardCharsets; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import java.util.ArrayList; import com.iqser.red.service.redaction.v1.server.redaction.utils.PositionUtil;
import java.util.Arrays; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import java.util.Collections; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.Before; import org.junit.Before;
@ -38,19 +31,25 @@ import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import com.iqser.red.service.configuration.v1.api.model.Colors; import java.io.BufferedReader;
import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse; import java.io.ByteArrayInputStream;
import com.iqser.red.service.configuration.v1.api.model.RulesResponse; import java.io.IOException;
import com.iqser.red.service.configuration.v1.api.model.TypeResponse; import java.io.InputStream;
import com.iqser.red.service.configuration.v1.api.model.TypeResult; import java.io.InputStreamReader;
import com.iqser.red.service.redaction.v1.model.RedactionRequest; import java.net.URL;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import java.nio.charset.StandardCharsets;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import java.util.ArrayList;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import java.util.Arrays;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import java.util.Collections;
import com.iqser.red.service.redaction.v1.server.redaction.utils.PositionUtil; import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import java.util.List;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
@SpringBootTest @SpringBootTest
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@ -79,6 +78,8 @@ public class EntityRedactionServiceTest {
@Autowired @Autowired
private DroolsExecutionService droolsExecutionService; private DroolsExecutionService droolsExecutionService;
private final static String TEST_RULESET_ID = "123";
@TestConfiguration @TestConfiguration
public static class RedactionIntegrationTestConfiguration { public static class RedactionIntegrationTestConfiguration {
@ -129,21 +130,21 @@ public class EntityRedactionServiceTest {
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H.")) .entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))
.build(); .build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")) .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) { try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
} }
@ -162,20 +163,20 @@ public class EntityRedactionServiceTest {
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H.")) .entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))
.build(); .build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")) .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) { try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
} }
@ -187,22 +188,22 @@ public class EntityRedactionServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" + ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" +
" Supplement - Identity of the active substance - Reference list.pdf"); " Supplement - Identity of the active substance - Reference list.pdf");
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))) .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))) .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities() assertThat(classifiedDoc.getEntities()
.entrySet() .entrySet()
.stream() .stream()
@ -212,7 +213,7 @@ public class EntityRedactionServiceTest {
"the plant protection product.pdf"); "the plant protection product.pdf");
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities() assertThat(classifiedDoc.getEntities()
.entrySet() .entrySet()
.stream() .stream()
@ -224,22 +225,22 @@ public class EntityRedactionServiceTest {
public void testFalsePositiveInWrongCell() throws IOException { public void testFalsePositiveInWrongCell() throws IOException {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))) .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))) .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))) .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream() assertThat(classifiedDoc.getEntities().get(1).stream()
.filter(entity -> entity.getMatchedRule() == 9) .filter(entity -> entity.getMatchedRule() == 9)
@ -288,27 +289,27 @@ public class EntityRedactionServiceTest {
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + "\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
" section.redactBetween(\"Contact:\", \"Tel.:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " section.redactBetween(\"Contact:\", \"Tel.:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
" end"; " end";
when(rulesClient.getVersion()).thenReturn(RULES_VERSION.incrementAndGet()); when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules)); when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
droolsExecutionService.updateRules(); droolsExecutionService.updateRules(TEST_RULESET_ID);
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))) .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))) .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream() assertThat(classifiedDoc.getEntities().get(1).stream()
.filter(entity -> entity.getMatchedRule() == 6) .filter(entity -> entity.getMatchedRule() == 6)
@ -329,27 +330,27 @@ public class EntityRedactionServiceTest {
"Section(searchText.toLowerCase().contains(\"batches produced at\"))\n" + " then\n" + " section" + "Section(searchText.toLowerCase().contains(\"batches produced at\"))\n" + " then\n" + " section" +
".redactIfPrecededBy(\"batches produced at\", \"sponsor\", 11, \"Redacted because it represents a " + ".redactIfPrecededBy(\"batches produced at\", \"sponsor\", 11, \"Redacted because it represents a " +
"sponsor company\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " end"; "sponsor company\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " end";
when(rulesClient.getVersion()).thenReturn(RULES_VERSION.incrementAndGet()); when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules)); when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
droolsExecutionService.updateRules(); droolsExecutionService.updateRules(TEST_RULESET_ID);
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf");
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse authorResponse = DictionaryResponse.builder() DictionaryResponse authorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(authorResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(authorResponse);
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))) .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream() assertThat(classifiedDoc.getEntities().get(1).stream()
.filter(entity -> entity.getMatchedRule() == 11) .filter(entity -> entity.getMatchedRule() == 11)
@ -368,20 +369,20 @@ public class EntityRedactionServiceTest {
.entries(Arrays.asList("Bissig R.", "Thanei P.")) .entries(Arrays.asList("Bissig R.", "Thanei P."))
.build(); .build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")) .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8);
assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y
@ -393,16 +394,16 @@ public class EntityRedactionServiceTest {
.entries(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C.")) .entries(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C."))
.build(); .build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
addressResponse = DictionaryResponse.builder() addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")) .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3);
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9);
@ -419,20 +420,20 @@ public class EntityRedactionServiceTest {
.entries(Collections.singletonList("Aldershof S.")) .entries(Collections.singletonList("Aldershof S."))
.build(); .build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")) .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList()) .entries(Collections.emptyList())
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null); entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6);
} }
@ -464,27 +465,27 @@ public class EntityRedactionServiceTest {
" 1107/2009 Art. 63 (2g)\");\n" + " 1107/2009 Art. 63 (2g)\");\n" +
" section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" + " section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" +
" end"; " end";
when(rulesClient.getVersion()).thenReturn(RULES_VERSION.incrementAndGet()); when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules)); when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
TypeResponse typeResponse = TypeResponse.builder() TypeResponse typeResponse = TypeResponse.builder()
.types(Arrays.asList( .types(Arrays.asList(
TypeResult.builder().type(AUTHOR_CODE).hexColor("#ffff00").build(), TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(AUTHOR_CODE).hexColor("#ffff00").build(),
TypeResult.builder().type(ADDRESS_CODE).hexColor("#ff00ff").build(), TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(ADDRESS_CODE).hexColor("#ff00ff").build(),
TypeResult.builder().type(SPONSOR_CODE).hexColor("#00ffff").build())) TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(SPONSOR_CODE).hexColor("#00ffff").build()))
.build(); .build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getAllTypes()).thenReturn(typeResponse); when(dictionaryClient.getAllTypes(TEST_RULESET_ID)).thenReturn(typeResponse);
// Default empty return to prevent NPEs // Default empty return to prevent NPEs
DictionaryResponse dictionaryResponse = DictionaryResponse.builder() DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.build(); .build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse); when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder() DictionaryResponse addressResponse = DictionaryResponse.builder()
.build(); .build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder() DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.build(); .build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
Colors colors = new Colors(); Colors colors = new Colors();
colors.setDefaultColor("#acfc00"); colors.setDefaultColor("#acfc00");
@ -492,7 +493,7 @@ public class EntityRedactionServiceTest {
colors.setRequestAdd("#04b093"); colors.setRequestAdd("#04b093");
colors.setRequestRemove("#04b093"); colors.setRequestRemove("#04b093");
when(dictionaryClient.getColors()).thenReturn(colors); when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
} }
@ -516,7 +517,7 @@ public class EntityRedactionServiceTest {
@Test @Test
public void testAuthorSplitting(){ public void testAuthorSplitting() {
String word = "Porch JR, " + "Kendall TZ, " + "Krueger HO"; String word = "Porch JR, " + "Kendall TZ, " + "Krueger HO";
@ -530,8 +531,8 @@ public class EntityRedactionServiceTest {
allMatches.add(matcher.group()); allMatches.add(matcher.group());
} }
for(String name: allMatches) { for (String name : allMatches) {
if(name.length() >= 3) { if (name.length() >= 3) {
System.out.println(name); System.out.println(name);
// dictionaryService.addToLocalDictionary(type, name); // dictionaryService.addToLocalDictionary(type, name);
} }