Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #3: Load initial rules from configuration service.

Browse Source 
Merge in RED/redaction-service from feature/load-initial-rules-from-configuration-service to master

* commit '01d08fb1913d748fe04fcea78b8a405a92bd1a49':
  Removed code is debugged code.
  Add real test.
  Remove unused import.
  Make testing possible again.
  Move update of rules out of controller.
  Load initial rules from configuration service.
This commit is contained in:
Thierry Goeckel 2020-07-17 11:24:07 +02:00
commit 0ed8530cb5
10 changed files with 132 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java
View File

@ -1,7 +1,6 @@
package com.iqser.red.service.redaction.v1.resources;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@ -24,9 +23,6 @@ public interface RedactionResource {
@PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
@GetMapping(value = "/rules", produces = MediaType.APPLICATION_JSON_VALUE)
String getRules();
@PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE)
void updateRules(@RequestBody String rules);

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java
View File

@ -1,11 +1,15 @@
package com.iqser.red.service.redaction.v1.server;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.kie.internal.io.ResourceFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@ -15,6 +19,8 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@Import({DefaultWebMvcConfiguration.class})
@ -22,19 +28,22 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
public class Application {
@Autowired
private RulesClient rulesClient;
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
private static final String drlFile = "drools/rules.drl";
@Bean
public KieContainer kieContainer() {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.write(ResourceFactory.newClassPathResource(drlFile));
RulesResponse rules = rulesClient.getRules();
InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();

23
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -13,7 +13,6 @@ import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
@ -36,15 +35,11 @@ public class RedactionController implements RedactionResource {
private final EntityRedactionService entityRedactionService;
private final PdfFlattenService pdfFlattenService;
private final DroolsExecutionService droolsExecutionService;
private final RulesClient rulesClient;
private long rulesVersion = -1;
@Override
public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) {
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
updateRules();
pdDocument.setAllSecurityToBeRemoved(true);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@ -68,7 +63,6 @@ public class RedactionController implements RedactionResource {
public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
updateRules();
pdDocument.setAllSecurityToBeRemoved(true);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@ -86,7 +80,6 @@ public class RedactionController implements RedactionResource {
public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
updateRules();
pdDocument.setAllSecurityToBeRemoved(true);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@ -104,7 +97,6 @@ public class RedactionController implements RedactionResource {
public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
updateRules();
pdDocument.setAllSecurityToBeRemoved(true);
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@ -127,11 +119,6 @@ public class RedactionController implements RedactionResource {
}
@Override
public String getRules() {
return droolsExecutionService.getRules();
}
@Override
public void updateRules(@RequestBody String rules) {
droolsExecutionService.updateRules(rules);
@ -149,14 +136,4 @@ public class RedactionController implements RedactionResource {
}
private void updateRules() {
long version = rulesClient.getVersion();
if (version > rulesVersion) {
rulesVersion = version;
updateRules(rulesClient.getRules().getRules());
}
}
}

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -4,8 +4,6 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
@ -16,30 +14,45 @@ import org.kie.api.runtime.KieSession;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class DroolsExecutionService {
private final RulesClient rulesClient;
@Autowired
private KieContainer kieContainer;
private String currentDrlRules;
@PostConstruct
public void init() {
currentDrlRules = ResourceLoader.loadAsString("drools/rules.drl");
}
private long rulesVersion = -1;
public Section executeRules(Section section) {
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("section", section);
kieSession.insert(section);
kieSession.fireAllRules();
kieSession.dispose();
return section;
}
public void updateRules() {
long version = rulesClient.getVersion();
if (version > rulesVersion) {
rulesVersion = version;
updateRules(rulesClient.getRules().getRules());
}
}
public void updateRules(String drlAsString) {
@ -51,7 +64,8 @@ public class DroolsExecutionService {
KieServices kieServices = KieServices.Factory.get();
InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8));
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
.newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();
@ -64,6 +78,7 @@ public class DroolsExecutionService {
}
public String getRules() {
return currentDrlRules;
}

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -27,10 +27,10 @@ public class EntityRedactionService {
private final DictionaryService dictionaryService;
private final DroolsExecutionService droolsExecutionService;
public void processDocument(Document classifiedDoc) {
dictionaryService.updateDictionary();
droolsExecutionService.updateRules();
Set<Entity> documentEntities = new HashSet<>();
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
@ -95,7 +95,6 @@ public class EntityRedactionService {
});
}
private Set<Entity> findEntities(SearchableText searchableText) {
String normalizedInputString = searchableText.toString();
@ -127,7 +126,6 @@ public class EntityRedactionService {
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
}
public void removeEntitiesContainedInLarger(Set<Entity> entities) {
List<Entity> wordsToRemove = new ArrayList<>();
for (Entity word : entities) {
@ -140,5 +138,4 @@ public class EntityRedactionService {
entities.removeAll(wordsToRemove);
}
}
}

33
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java
View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
@ -20,38 +19,12 @@ public class ResourceLoader {
if (resource == null) {
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
}
try (InputStream is = resource.openStream();
InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
BufferedReader br = new BufferedReader(isr)) {
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
return br.lines().collect(Collectors.toSet());
} catch (IOException e) {
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
}
}
public String loadAsString(String classpathPath) {
URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
if (resource == null) {
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
}
try (InputStream is = resource.openStream();
InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
BufferedReader br = new BufferedReader(isr)) {
StringBuffer sb = new StringBuffer();
String str;
while ((str = br.readLine()) != null) {
sb.append(str).append("\n");
}
return sb.toString();
} catch (IOException e) {
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
}
}
}
}

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DummyTest.java
View File

@ -1,14 +0,0 @@
package com.iqser.red.service.redaction.v1.server;
import org.junit.Test;
/**
*
*/
public class DummyTest {
@Test
public void dummy(){
System.out.println("Hello World");
}
}

73
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,22 +1,40 @@
package com.iqser.red.service.redaction.v1.server;
import static org.mockito.Mockito.when;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.test.context.junit4.SpringRunner;
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
@Ignore
@ -27,7 +45,37 @@ public class RedactionIntegrationTest {
@Autowired
private RedactionController redactionController;
@MockBean
private RulesClient rulesClient;
@TestConfiguration
public static class RedactionIntegrationTestConfiguration {
@Bean
public KieContainer kieContainer() {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
InputStream input = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();
return kieServices.newKieContainer(kieModule.getReleaseId());
}
}
@Before
public void stubRulesClient() {
when(rulesClient.getVersion()).thenReturn(0L);
when(rulesClient.getRules()).thenReturn(new RulesResponse(loadFromClassPath("drools/rules.drl")));
}
@Test
public void redactionTest() throws IOException {
@ -49,7 +97,6 @@ public class RedactionIntegrationTest {
System.out.println("numberOfPages: " + result.getNumberOfPages());
}
@Test
public void classificationTest() throws IOException {
@ -64,7 +111,6 @@ public class RedactionIntegrationTest {
}
}
@Test
public void sectionsTest() throws IOException {
@ -107,4 +153,25 @@ public class RedactionIntegrationTest {
}
}
private String loadFromClassPath(String path) {
URL resource = ResourceLoader.class.getClassLoader()
.getResource(path);
if (resource == null) {
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
}
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
StringBuilder sb = new StringBuilder();
String str;
while ((str = br.readLine()) != null) {
sb.append(str)
.append("\n");
}
return sb.toString();
} catch (IOException e) {
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
}
}
}

21
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java Normal file
View File

@ -0,0 +1,21 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils;
import org.assertj.core.api.Assertions;
import org.junit.Test;
public class TextNormalizationUtilitiesTest {
@Test
public void testHyphenRemoval() {
String test = "Without these peo-\nple, this conference would not happen";
Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
.contains("\npeople");
test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
.contains("\neinzelnen", "\nVertrages");
}
}

0
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/rules.drl → redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File