diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index 464b5799..2ce17e56 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -1,7 +1,6 @@ package com.iqser.red.service.redaction.v1.resources; import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -24,9 +23,6 @@ public interface RedactionResource { @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest); - @GetMapping(value = "/rules", produces = MediaType.APPLICATION_JSON_VALUE) - String getRules(); - @PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE) void updateRules(@RequestBody String rules); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java index bb091c47..1b4f237b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java @@ -1,11 +1,15 @@ package com.iqser.red.service.redaction.v1.server; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + import org.kie.api.KieServices; import org.kie.api.builder.KieBuilder; import org.kie.api.builder.KieFileSystem; import org.kie.api.builder.KieModule; import org.kie.api.runtime.KieContainer; -import org.kie.internal.io.ResourceFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -15,6 +19,8 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration; +import com.iqser.red.service.configuration.v1.api.model.RulesResponse; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; @Import({DefaultWebMvcConfiguration.class}) @@ -22,19 +28,22 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) public class Application { + @Autowired + private RulesClient rulesClient; + public static void main(String[] args) { SpringApplication.run(Application.class, args); } - private static final String drlFile = "drools/rules.drl"; - @Bean public KieContainer kieContainer() { KieServices kieServices = KieServices.Factory.get(); KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); - kieFileSystem.write(ResourceFactory.newClassPathResource(drlFile)); + RulesResponse rules = rulesClient.getRules(); + InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8)); + kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input)); KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); kieBuilder.buildAll(); KieModule kieModule = kieBuilder.getKieModule(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index f46a600a..0927e8cc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -13,7 +13,6 @@ import com.iqser.red.service.redaction.v1.model.RedactionResult; import com.iqser.red.service.redaction.v1.resources.RedactionResource; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; -import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService; @@ -36,15 +35,11 @@ public class RedactionController implements RedactionResource { private final EntityRedactionService entityRedactionService; private final PdfFlattenService pdfFlattenService; private final DroolsExecutionService droolsExecutionService; - private final RulesClient rulesClient; - - private long rulesVersion = -1; @Override public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) { try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) { - updateRules(); pdDocument.setAllSecurityToBeRemoved(true); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); @@ -68,7 +63,6 @@ public class RedactionController implements RedactionResource { public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) { try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) { - updateRules(); pdDocument.setAllSecurityToBeRemoved(true); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); @@ -86,7 +80,6 @@ public class RedactionController implements RedactionResource { public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) { try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) { - updateRules(); pdDocument.setAllSecurityToBeRemoved(true); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); @@ -104,7 +97,6 @@ public class RedactionController implements RedactionResource { public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) { try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) { - updateRules(); pdDocument.setAllSecurityToBeRemoved(true); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); @@ -127,11 +119,6 @@ public class RedactionController implements RedactionResource { } - @Override - public String getRules() { - return droolsExecutionService.getRules(); - } - @Override public void updateRules(@RequestBody String rules) { droolsExecutionService.updateRules(rules); @@ -149,14 +136,4 @@ public class RedactionController implements RedactionResource { } - private void updateRules() { - - long version = rulesClient.getVersion(); - if (version > rulesVersion) { - rulesVersion = version; - updateRules(rulesClient.getRules().getRules()); - } - - } - } \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 3035b992..95b4036b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -4,8 +4,6 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import javax.annotation.PostConstruct; - import org.apache.commons.lang3.StringUtils; import org.kie.api.KieServices; import org.kie.api.builder.KieBuilder; @@ -16,30 +14,45 @@ import org.kie.api.runtime.KieSession; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; import com.iqser.red.service.redaction.v1.server.redaction.model.Section; -import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; + +import lombok.RequiredArgsConstructor; @Service +@RequiredArgsConstructor public class DroolsExecutionService { + private final RulesClient rulesClient; + @Autowired private KieContainer kieContainer; private String currentDrlRules; - @PostConstruct - public void init() { - currentDrlRules = ResourceLoader.loadAsString("drools/rules.drl"); - } + private long rulesVersion = -1; public Section executeRules(Section section) { + KieSession kieSession = kieContainer.newKieSession(); kieSession.setGlobal("section", section); kieSession.insert(section); kieSession.fireAllRules(); kieSession.dispose(); + return section; + + } + + public void updateRules() { + + long version = rulesClient.getVersion(); + if (version > rulesVersion) { + rulesVersion = version; + updateRules(rulesClient.getRules().getRules()); + } + } public void updateRules(String drlAsString) { @@ -51,7 +64,8 @@ public class DroolsExecutionService { KieServices kieServices = KieServices.Factory.get(); InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8)); KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); - kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input)); + kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources() + .newInputStreamResource(input)); KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); kieBuilder.buildAll(); KieModule kieModule = kieBuilder.getKieModule(); @@ -64,6 +78,7 @@ public class DroolsExecutionService { } public String getRules() { + return currentDrlRules; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index f2f65df5..a76d775e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -27,10 +27,10 @@ public class EntityRedactionService { private final DictionaryService dictionaryService; private final DroolsExecutionService droolsExecutionService; - public void processDocument(Document classifiedDoc) { dictionaryService.updateDictionary(); + droolsExecutionService.updateRules(); Set documentEntities = new HashSet<>(); for (Paragraph paragraph : classifiedDoc.getParagraphs()) { @@ -95,7 +95,6 @@ public class EntityRedactionService { }); } - private Set findEntities(SearchableText searchableText) { String normalizedInputString = searchableText.toString(); @@ -127,7 +126,6 @@ public class EntityRedactionService { return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’'; } - public void removeEntitiesContainedInLarger(Set entities) { List wordsToRemove = new ArrayList<>(); for (Entity word : entities) { @@ -140,5 +138,4 @@ public class EntityRedactionService { entities.removeAll(wordsToRemove); } - -} +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java index 918320bd..5fc33bbd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -20,38 +19,12 @@ public class ResourceLoader { if (resource == null) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); } - try (InputStream is = resource.openStream(); - InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); - BufferedReader br = new BufferedReader(isr)) { + try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { return br.lines().collect(Collectors.toSet()); } catch (IOException e) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); } + } - - public String loadAsString(String classpathPath) { - URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath); - if (resource == null) { - throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); - } - try (InputStream is = resource.openStream(); - InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); - BufferedReader br = new BufferedReader(isr)) { - StringBuffer sb = new StringBuffer(); - String str; - while ((str = br.readLine()) != null) { - sb.append(str).append("\n"); - } - return sb.toString(); - } catch (IOException e) { - throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); - } - } -} - - - - - - +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DummyTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DummyTest.java deleted file mode 100644 index f791129f..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DummyTest.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.iqser.red.service.redaction.v1.server; - -import org.junit.Test; - -/** - * - */ -public class DummyTest { - - @Test - public void dummy(){ - System.out.println("Hello World"); - } -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 62c8f6ed..fdc58d8f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1,22 +1,40 @@ package com.iqser.red.service.redaction.v1.server; - +import static org.mockito.Mockito.when; import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; import org.apache.commons.io.IOUtils; +import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; +import org.kie.api.KieServices; +import org.kie.api.builder.KieBuilder; +import org.kie.api.builder.KieFileSystem; +import org.kie.api.builder.KieModule; +import org.kie.api.runtime.KieContainer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.ResourceLoader; import org.springframework.test.context.junit4.SpringRunner; +import com.iqser.red.service.configuration.v1.api.model.RulesResponse; import com.iqser.red.service.redaction.v1.model.RedactionRequest; import com.iqser.red.service.redaction.v1.model.RedactionResult; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; @Ignore @@ -27,7 +45,37 @@ public class RedactionIntegrationTest { @Autowired private RedactionController redactionController; + @MockBean + private RulesClient rulesClient; + @TestConfiguration + public static class RedactionIntegrationTestConfiguration { + + @Bean + public KieContainer kieContainer() { + + KieServices kieServices = KieServices.Factory.get(); + + KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); + InputStream input = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); + kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input)); + KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); + kieBuilder.buildAll(); + KieModule kieModule = kieBuilder.getKieModule(); + + return kieServices.newKieContainer(kieModule.getReleaseId()); + + } + + } + + @Before + public void stubRulesClient() { + + when(rulesClient.getVersion()).thenReturn(0L); + when(rulesClient.getRules()).thenReturn(new RulesResponse(loadFromClassPath("drools/rules.drl"))); + + } @Test public void redactionTest() throws IOException { @@ -49,7 +97,6 @@ public class RedactionIntegrationTest { System.out.println("numberOfPages: " + result.getNumberOfPages()); } - @Test public void classificationTest() throws IOException { @@ -64,7 +111,6 @@ public class RedactionIntegrationTest { } } - @Test public void sectionsTest() throws IOException { @@ -107,4 +153,25 @@ public class RedactionIntegrationTest { } } + private String loadFromClassPath(String path) { + + URL resource = ResourceLoader.class.getClassLoader() + .getResource(path); + if (resource == null) { + throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); + } + try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String str; + while ((str = br.readLine()) != null) { + sb.append(str) + .append("\n"); + } + return sb.toString(); + } catch (IOException e) { + throw new IllegalArgumentException("could not load classpath resource: " + path, e); + } + + } + } \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java new file mode 100644 index 00000000..e4b04ae1 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.redaction.v1.server.redaction.utils; + +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public class TextNormalizationUtilitiesTest { + + @Test + public void testHyphenRemoval() { + + String test = "Without these peo-\nple, this conference would not happen"; + Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test)) + .contains("\npeople"); + + test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss."; + Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test)) + .contains("\neinzelnen", "\nVertrages"); + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl similarity index 100% rename from redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/rules.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl