Pull request #5 : RED-155: Bump config service version to make sure, dictionaries and

Merge in RED/redaction-service from feature/RED-155 to master * commit 'c80cae3fc361a651e120626daf26d39de1dc656f': No need to write or add rules to file in classpath. RED-155: Bump config service version to make sure, dictionaries and rules are pulled correctly.
No need to write or add rules to file in classpath.
2020-07-17 13:01:32 +02:00 · 2020-07-17 12:55:49 +02:00 · 2020-07-17 12:40:49 +02:00 · 2020-07-17 11:24:07 +02:00 · 2020-07-17 11:13:31 +02:00 · 2020-07-16 15:57:22 +02:00
12 changed files with 161 additions and 89 deletions
@@ -23,9 +23,6 @@ public interface RedactionResource {
    @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
    RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);

-    @PostMapping(value = "/rules", produces = MediaType.APPLICATION_JSON_VALUE)
-    String getRules();
-
    @PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE)
    void updateRules(@RequestBody String rules);

@@ -31,6 +31,16 @@
    </dependencyManagement>

    <dependencies>
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>redaction-service-api-v1</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>configuration-service-api-v1</artifactId>
+            <version>1.0.0</version>
+        </dependency>
        <dependency>
            <groupId>org.drools</groupId>
            <artifactId>drools-core</artifactId>
@@ -46,11 +56,6 @@
            <artifactId>jts-core</artifactId>
            <version>1.16.1</version>
        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>redaction-service-api-v1</artifactId>
-            <version>${project.version}</version>
-        </dependency>
        <!-- commons -->
        <dependency>
            <groupId>com.iqser.gin4.commons</groupId>
@@ -1,11 +1,15 @@
 package com.iqser.red.service.redaction.v1.server;

+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
-import org.kie.internal.io.ResourceFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -15,6 +19,8 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

 import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

@Import({DefaultWebMvcConfiguration.class})
@@ -22,19 +28,22 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

+    @Autowired
+    private RulesClient rulesClient;
+
    public static void main(String[] args) {
        SpringApplication.run(Application.class, args);
    }

-    private static final String drlFile = "drools/rules.drl";
-
    @Bean
    public KieContainer kieContainer() {

        KieServices kieServices = KieServices.Factory.get();

        KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-        kieFileSystem.write(ResourceFactory.newClassPathResource(drlFile));
+        RulesResponse rules = rulesClient.getRules();
+        InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
+        kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
        KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
        kieBuilder.buildAll();
        KieModule kieModule = kieBuilder.getKieModule();
@@ -0,0 +1,9 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
+
+@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
+public interface RulesClient extends RulesResource {
+}
@@ -24,9 +24,7 @@ import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlatte
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;

 import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;

-@Slf4j
@RestController
@RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
@@ -38,7 +36,7 @@ public class RedactionController implements RedactionResource {
    private final PdfFlattenService pdfFlattenService;
    private final DroolsExecutionService droolsExecutionService;

-
+    @Override
    public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) {

        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
@@ -58,9 +56,10 @@ public class RedactionController implements RedactionResource {
        } catch (IOException e) {
            throw new RedactionException(e);
        }
+
    }

-
+    @Override
    public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {

        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
@@ -74,9 +73,10 @@ public class RedactionController implements RedactionResource {
        } catch (IOException e) {
            throw new RedactionException(e);
        }
+
    }

-
+    @Override
    public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {

        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
@@ -90,10 +90,12 @@ public class RedactionController implements RedactionResource {
        } catch (IOException e) {
            throw new RedactionException(e);
        }
+
    }

-
+    @Override
    public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
+
        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
            pdDocument.setAllSecurityToBeRemoved(true);

@@ -114,17 +116,16 @@ public class RedactionController implements RedactionResource {
        } catch (IOException e) {
            throw new RedactionException(e);
        }
+
    }

-    public String getRules() {
-        return droolsExecutionService.getRules();
-    }
-
+    @Override
    public void updateRules(@RequestBody String rules) {
        droolsExecutionService.updateRules(rules);
    }

    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
+
        try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
            document.save(byteArrayOutputStream);
            return RedactionResult.builder()
@@ -132,6 +133,7 @@ public class RedactionController implements RedactionResource {
                    .numberOfPages(numberOfPages)
                    .build();
        }
+
    }

 }
@@ -4,8 +4,6 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;

-import javax.annotation.PostConstruct;
-
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
@@ -16,30 +14,43 @@ import org.kie.api.runtime.KieSession;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;

+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+
+import lombok.RequiredArgsConstructor;

@Service
+@RequiredArgsConstructor
 public class DroolsExecutionService {

+    private final RulesClient rulesClient;
+
    @Autowired
    private KieContainer kieContainer;

-    private String currentDrlRules;
-
-    @PostConstruct
-    public void init() {
-        currentDrlRules = ResourceLoader.loadAsString("drools/rules.drl");
-    }
+    private long rulesVersion = -1;

    public Section executeRules(Section section) {
+
        KieSession kieSession = kieContainer.newKieSession();
        kieSession.setGlobal("section", section);
        kieSession.insert(section);
        kieSession.fireAllRules();
        kieSession.dispose();
+
        return section;
+
+    }
+
+    public void updateRules() {
+
+        long version = rulesClient.getVersion();
+        if (version > rulesVersion) {
+            rulesVersion = version;
+            updateRules(rulesClient.getRules().getRules());
+        }
+
    }

    public void updateRules(String drlAsString) {
@@ -51,20 +62,15 @@ public class DroolsExecutionService {
            KieServices kieServices = KieServices.Factory.get();
            InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8));
            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            kieFileSystem.write(kieServices.getResources().newInputStreamResource(input));
            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
            kieBuilder.buildAll();
            KieModule kieModule = kieBuilder.getKieModule();
            kieContainer.updateToVersion(kieModule.getReleaseId());
-            currentDrlRules = drlAsString;
        } catch (Exception e) {
            throw new RulesValidationException("Could not update rules", e);
        }

    }

-    public String getRules() {
-        return currentDrlRules;
-    }
-
 }
@@ -27,10 +27,10 @@ public class EntityRedactionService {
    private final DictionaryService dictionaryService;
    private final DroolsExecutionService droolsExecutionService;

-
    public void processDocument(Document classifiedDoc) {

        dictionaryService.updateDictionary();
+        droolsExecutionService.updateRules();

        Set<Entity> documentEntities = new HashSet<>();
        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
@@ -95,7 +95,6 @@ public class EntityRedactionService {
        });
    }

-
    private Set<Entity> findEntities(SearchableText searchableText) {

        String normalizedInputString = searchableText.toString();
@@ -127,7 +126,6 @@ public class EntityRedactionService {
        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
    }

-
    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : entities) {
@@ -140,5 +138,4 @@ public class EntityRedactionService {
        entities.removeAll(wordsToRemove);
    }

-
-}
+}
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;

 import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@@ -20,38 +19,12 @@ public class ResourceLoader {
        if (resource == null) {
            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
        }
-        try (InputStream is = resource.openStream();
-             InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
-             BufferedReader br = new BufferedReader(isr)) {
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
            return br.lines().collect(Collectors.toSet());
        } catch (IOException e) {
            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
        }
+
    }

-
-    public String loadAsString(String classpathPath) {
-        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
-        }
-        try (InputStream is = resource.openStream();
-             InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
-             BufferedReader br = new BufferedReader(isr)) {
-            StringBuffer sb = new StringBuffer();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
-        }
-    }
-}
-
-
-
-
-
-
+}
@@ -1,14 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import org.junit.Test;
-
-/**
- *
- */
-public class DummyTest {
-
-    @Test
-    public void dummy(){
-        System.out.println("Hello World");
-    }
-}
@@ -1,22 +1,40 @@
 package com.iqser.red.service.redaction.v1.server;

-
+import static org.mockito.Mockito.when;
 import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT;

+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;

 import org.apache.commons.io.IOUtils;
+import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.ResourceLoader;
 import org.springframework.test.context.junit4.SpringRunner;

+import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;

@Ignore
@@ -27,7 +45,37 @@ public class RedactionIntegrationTest {
    @Autowired
    private RedactionController redactionController;

+    @MockBean
+    private RulesClient rulesClient;

+    @TestConfiguration
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            InputStream input = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+
+        }
+
+    }
+
+    @Before
+    public void stubRulesClient() {
+
+        when(rulesClient.getVersion()).thenReturn(0L);
+        when(rulesClient.getRules()).thenReturn(new RulesResponse(loadFromClassPath("drools/rules.drl")));
+
+    }

    @Test
    public void redactionTest() throws IOException {
@@ -49,7 +97,6 @@ public class RedactionIntegrationTest {
        System.out.println("numberOfPages: " + result.getNumberOfPages());
    }

-
    @Test
    public void classificationTest() throws IOException {

@@ -64,7 +111,6 @@ public class RedactionIntegrationTest {
        }
    }

-
    @Test
    public void sectionsTest() throws IOException {

@@ -107,4 +153,25 @@ public class RedactionIntegrationTest {
        }
    }

+    private String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader()
+                .getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str)
+                        .append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+
+    }
+
 }
@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+public class TextNormalizationUtilitiesTest {
+
+    @Test
+    public void testHyphenRemoval() {
+
+        String test = "Without these peo-\nple, this conference would not happen";
+        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
+                .contains("\npeople");
+
+        test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
+        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
+                .contains("\neinzelnen", "\nVertrages");
+
+    }
+
+}
Author	SHA1	Message	Date
Cheng Zhu	fa7693e88b	Pull request #5 : RED-155: Bump config service version to make sure, dictionaries and Merge in RED/redaction-service from feature/RED-155 to master * commit 'c80cae3fc361a651e120626daf26d39de1dc656f': No need to write or add rules to file in classpath. RED-155: Bump config service version to make sure, dictionaries and rules are pulled correctly.	2020-07-17 13:01:32 +02:00
Thierry Göckel	c80cae3fc3	No need to write or add rules to file in classpath.	2020-07-17 12:55:49 +02:00
Thierry Göckel	4832555343	RED-155: Bump config service version to make sure, dictionaries and rules are pulled correctly.	2020-07-17 12:40:49 +02:00
Thierry Goeckel	0ed8530cb5	Pull request #3 : Load initial rules from configuration service. Merge in RED/redaction-service from feature/load-initial-rules-from-configuration-service to master * commit '01d08fb1913d748fe04fcea78b8a405a92bd1a49': Removed code is debugged code. Add real test. Remove unused import. Make testing possible again. Move update of rules out of controller. Load initial rules from configuration service.	2020-07-17 11:24:07 +02:00
Thierry Göckel	01d08fb191	Removed code is debugged code.	2020-07-17 11:13:31 +02:00
Thierry Göckel	705c499911	Add real test.	2020-07-16 15:57:22 +02:00
Thierry Göckel	7435c1eb87	Remove unused import.	2020-07-16 14:18:18 +02:00
Thierry Göckel	531e34c6d0	Make testing possible again.	2020-07-16 13:37:43 +02:00
Thierry Göckel	cc0d585c0b	Move update of rules out of controller.	2020-07-16 10:21:23 +02:00
Thierry Göckel	74e5bc0635	Load initial rules from configuration service.	2020-07-14 17:12:27 +02:00
Cheng Zhu	fbeaebab7d	Pull request #2 : DEV: Update rules on each redaction request. Merge in RED/redaction-service from dev/update-rules-on-redaction-requests to master * commit 'bf89e42fcf597033a17acc63919f62aed6640285': DEV: Update rules on each redaction request.	2020-07-09 22:50:14 +02:00
Thierry Göckel	bf89e42fcf	DEV: Update rules on each redaction request.	2020-07-09 13:47:31 +02:00
Thierry Goeckel	a3d471e940	Pull request #1 : DEV: Handle rules validation exception when updating. Merge in RED/redaction-service from dev/rules-validation to master * commit '23741deff713d5c865c13aa3a8cac73795d969e8': DEV: Use appropriate HTTP request type. DEV: Handle rules validation exception when updating.	2020-07-09 10:25:26 +02:00
Thierry Göckel	23741deff7	DEV: Use appropriate HTTP request type.	2020-07-09 10:06:17 +02:00