Pull request #3: Load initial rules from configuration service.

Merge in RED/redaction-service from feature/load-initial-rules-from-configuration-service to master

* commit '01d08fb1913d748fe04fcea78b8a405a92bd1a49':
  Removed code is debugged code.
  Add real test.
  Remove unused import.
  Make testing possible again.
  Move update of rules out of controller.
  Load initial rules from configuration service.

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.resources;
 
 import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
@@ -24,9 +23,6 @@ public interface RedactionResource {
     @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
     RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
 
-    @GetMapping(value = "/rules", produces = MediaType.APPLICATION_JSON_VALUE)
-    String getRules();
-
     @PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE)
     void updateRules(@RequestBody String rules);
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -1,11 +1,15 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
-import org.kie.internal.io.ResourceFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -15,6 +19,8 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
 import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 @Import({DefaultWebMvcConfiguration.class})
@@ -22,19 +28,22 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {
 
+    @Autowired
+    private RulesClient rulesClient;
+
     public static void main(String[] args) {
         SpringApplication.run(Application.class, args);
     }
 
-    private static final String drlFile = "drools/rules.drl";
-
     @Bean
     public KieContainer kieContainer() {
 
         KieServices kieServices = KieServices.Factory.get();
 
         KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-        kieFileSystem.write(ResourceFactory.newClassPathResource(drlFile));
+        RulesResponse rules = rulesClient.getRules();
+        InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
+        kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
         KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
         kieBuilder.buildAll();
         KieModule kieModule = kieBuilder.getKieModule();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -13,7 +13,6 @@ import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
@@ -36,15 +35,11 @@ public class RedactionController implements RedactionResource {
     private final EntityRedactionService entityRedactionService;
     private final PdfFlattenService pdfFlattenService;
     private final DroolsExecutionService droolsExecutionService;
-    private final RulesClient rulesClient;
-
-    private long rulesVersion = -1;
 
     @Override
     public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) {
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
-            updateRules();
             pdDocument.setAllSecurityToBeRemoved(true);
 
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@@ -68,7 +63,6 @@ public class RedactionController implements RedactionResource {
     public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
-            updateRules();
             pdDocument.setAllSecurityToBeRemoved(true);
 
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@@ -86,7 +80,6 @@ public class RedactionController implements RedactionResource {
     public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
-            updateRules();
             pdDocument.setAllSecurityToBeRemoved(true);
 
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@@ -104,7 +97,6 @@ public class RedactionController implements RedactionResource {
     public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
-            updateRules();
             pdDocument.setAllSecurityToBeRemoved(true);
 
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@@ -127,11 +119,6 @@ public class RedactionController implements RedactionResource {
 
     }
 
-    @Override
-    public String getRules() {
-        return droolsExecutionService.getRules();
-    }
-
     @Override
     public void updateRules(@RequestBody String rules) {
         droolsExecutionService.updateRules(rules);
@@ -149,14 +136,4 @@ public class RedactionController implements RedactionResource {
 
     }
 
-    private void updateRules() {
-
-        long version = rulesClient.getVersion();
-        if (version > rulesVersion) {
-            rulesVersion = version;
-            updateRules(rulesClient.getRules().getRules());
-        }
-
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -4,8 +4,6 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 
-import javax.annotation.PostConstruct;
-
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
@@ -16,30 +14,45 @@ import org.kie.api.runtime.KieSession;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+
+import lombok.RequiredArgsConstructor;
 
 @Service
+@RequiredArgsConstructor
 public class DroolsExecutionService {
 
+    private final RulesClient rulesClient;
+
     @Autowired
     private KieContainer kieContainer;
 
     private String currentDrlRules;
 
-    @PostConstruct
+    private long rulesVersion = -1;
-    public void init() {
-        currentDrlRules = ResourceLoader.loadAsString("drools/rules.drl");
-    }
 
     public Section executeRules(Section section) {
+
         KieSession kieSession = kieContainer.newKieSession();
         kieSession.setGlobal("section", section);
         kieSession.insert(section);
         kieSession.fireAllRules();
         kieSession.dispose();
+
         return section;
+
+    }
+
+    public void updateRules() {
+
+        long version = rulesClient.getVersion();
+        if (version > rulesVersion) {
+            rulesVersion = version;
+            updateRules(rulesClient.getRules().getRules());
+        }
+
     }
 
     public void updateRules(String drlAsString) {
@@ -51,7 +64,8 @@ public class DroolsExecutionService {
             KieServices kieServices = KieServices.Factory.get();
             InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8));
             KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
+                    .newInputStreamResource(input));
             KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
             kieBuilder.buildAll();
             KieModule kieModule = kieBuilder.getKieModule();
@@ -64,6 +78,7 @@ public class DroolsExecutionService {
     }
 
     public String getRules() {
+
         return currentDrlRules;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -27,10 +27,10 @@ public class EntityRedactionService {
     private final DictionaryService dictionaryService;
     private final DroolsExecutionService droolsExecutionService;
 
-
     public void processDocument(Document classifiedDoc) {
 
         dictionaryService.updateDictionary();
+        droolsExecutionService.updateRules();
 
         Set<Entity> documentEntities = new HashSet<>();
         for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
@@ -95,7 +95,6 @@ public class EntityRedactionService {
         });
     }
 
-
     private Set<Entity> findEntities(SearchableText searchableText) {
 
         String normalizedInputString = searchableText.toString();
@@ -127,7 +126,6 @@ public class EntityRedactionService {
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
-
     public void removeEntitiesContainedInLarger(Set<Entity> entities) {
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
@@ -140,5 +138,4 @@ public class EntityRedactionService {
         entities.removeAll(wordsToRemove);
     }
 
-
+}
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
 import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@@ -20,38 +19,12 @@ public class ResourceLoader {
         if (resource == null) {
             throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
         }
-        try (InputStream is = resource.openStream();
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
-             InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
-             BufferedReader br = new BufferedReader(isr)) {
             return br.lines().collect(Collectors.toSet());
         } catch (IOException e) {
             throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
         }
+
     }
 
-
+}
-    public String loadAsString(String classpathPath) {
-        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
-        }
-        try (InputStream is = resource.openStream();
-             InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
-             BufferedReader br = new BufferedReader(isr)) {
-            StringBuffer sb = new StringBuffer();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
-        }
-    }
-}
-
-
-
-
-
-

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DummyTest.java

@@ -1,14 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import org.junit.Test;
-
-/**
- *
- */
-public class DummyTest {
-
-    @Test
-    public void dummy(){
-        System.out.println("Hello World");
-    }
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -1,22 +1,40 @@
 package com.iqser.red.service.redaction.v1.server;
 
-
+import static org.mockito.Mockito.when;
 import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT;
 
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
 
 import org.apache.commons.io.IOUtils;
+import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.ResourceLoader;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 
 @Ignore
@@ -27,7 +45,37 @@ public class RedactionIntegrationTest {
     @Autowired
     private RedactionController redactionController;
 
+    @MockBean
+    private RulesClient rulesClient;
 
+    @TestConfiguration
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            InputStream input = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+
+        }
+
+    }
+
+    @Before
+    public void stubRulesClient() {
+
+        when(rulesClient.getVersion()).thenReturn(0L);
+        when(rulesClient.getRules()).thenReturn(new RulesResponse(loadFromClassPath("drools/rules.drl")));
+
+    }
 
     @Test
     public void redactionTest() throws IOException {
@@ -49,7 +97,6 @@ public class RedactionIntegrationTest {
         System.out.println("numberOfPages: " + result.getNumberOfPages());
     }
 
-
     @Test
     public void classificationTest() throws IOException {
 
@@ -64,7 +111,6 @@ public class RedactionIntegrationTest {
         }
     }
 
-
     @Test
     public void sectionsTest() throws IOException {
 
@@ -107,4 +153,25 @@ public class RedactionIntegrationTest {
         }
     }
 
+    private String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader()
+                .getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str)
+                        .append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java

@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+public class TextNormalizationUtilitiesTest {
+
+    @Test
+    public void testHyphenRemoval() {
+
+        String test = "Without these peo-\nple, this conference would not happen";
+        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
+                .contains("\npeople");
+
+        test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
+        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
+                .contains("\neinzelnen", "\nVertrages");
+
+    }
+
+}