Pull request #394 : RED-3800 String Performance matching test

Merge in RED/redaction-service from RED-3800-string-performance-test to master * commit '21d717f0837c9cc9b2f5b251d372db88b25a7b6d': RED-3800 String Performance matching test RED-3800 String Performance matching test
RED-3800 String Performance matching test
2022-05-24 11:09:08 +02:00 · 2022-05-24 12:01:10 +03:00 · 2022-05-24 11:58:34 +03:00 · 2022-05-24 09:00:07 +02:00 · 2022-05-23 18:48:10 +02:00 · 2022-05-23 17:38:23 +02:00
12 changed files with 7323 additions and 1 deletions
@@ -98,9 +98,10 @@ public class EntityRedactionService {
                var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
                        .filter(idr -> idr.getRequestDate() != null)
-                        .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
+                        .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
                        .map(IdRemoval::getAnnotationId).collect(Collectors.toSet());

+
                if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
                        .isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
                    for (Image image : reanalysisSection.getImages()) {
@@ -166,6 +166,9 @@ public class EntitySearchUtils {

            if (existing.getType().equals(found.getType())) {
                existing.getEngines().addAll(found.getEngines());
+                existing.setLegalBasis(found.getLegalBasis());
+                existing.setMatchedRule(found.getMatchedRule());
+                existing.setRedactionReason(found.getRedactionReason());
                if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
                        || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
                    existing.setEntityType(EntityType.ENTITY);
@@ -230,6 +233,9 @@ public class EntitySearchUtils {
                }
                var existingEntity = existingOptional.get();
                existingEntity.getEngines().addAll(toAdd.getEngines());
+                existingEntity.setLegalBasis(toAdd.getLegalBasis());
+                existingEntity.setMatchedRule(toAdd.getMatchedRule());
+                existingEntity.setRedactionReason(toAdd.getRedactionReason());
            } else {
                existing.add(toAdd);
            }
@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server;

 import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
@@ -58,6 +59,7 @@ import java.io.*;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
 import java.util.*;
 import java.util.stream.Collectors;

@@ -715,6 +717,72 @@ public class RedactionIntegrationTest {
    }


+    @Test
+    public void testRemovePublishedInformations() throws IOException {
+
+        long start = System.currentTimeMillis();
+
+        ClassPathResource colorsResource = new ClassPathResource("colors/colors.json");
+        var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
+
+        ClassPathResource typeResource = new ClassPathResource("colors/types.json");
+        TypeReference<List<Type>> typeRefForTypes = new TypeReference<>(){};
+        List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
+
+
+        AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        ManualRedactions manualRedactions = new ManualRedactions();
+        manualRedactions.getIdsToRemove().add(IdRemoval.builder()
+                .annotationId("308dab9015bfafd911568cffe0a7f7de")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.of(2022,05,23,8,30,07,475479,  ZoneOffset.UTC))
+                        .processedDate(OffsetDateTime.of(2022,05,23,8,30,07,483651,  ZoneOffset.UTC))
+                .build());
+
+        manualRedactions.getForceRedactions().add(ManualForceRedaction.builder()
+                .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.of(2022,05,23,9,30,15,4653, ZoneOffset.UTC))
+                .processedDate(OffsetDateTime.of(2022,05,23,9,30,15,794,  ZoneOffset.UTC))
+                .build());
+
+        manualRedactions.getIdsToRemove().add(IdRemoval.builder()
+                .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.of(2022,05,23,8,30,23,961721,  ZoneOffset.UTC))
+                .processedDate(OffsetDateTime.of(2022,05,23,8,30,23,96528,  ZoneOffset.UTC))
+                .build());
+
+
+
+        request.setManualRedactions(manualRedactions);
+
+
+        AnalyzeResult result = analyzeService.analyze(request);
+
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                        .manualRedactions(manualRedactions)
+                        .colors(colors)
+                        .types(types)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
    @Test
    public void testTableRedaction() throws IOException {

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.annotate;

+import java.util.List;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -17,4 +21,6 @@ public class AnnotateRequest {
    private String dossierTemplateId;
    private String fileId;
    private ManualRedactions manualRedactions;
+    private Colors colors;
+    private List<Type> types;
 }
@@ -46,6 +46,8 @@ public class AnnotationService {
                .manualRedactions(annotateRequest.getManualRedactions())
                .dossierId(annotateRequest.getDossierId())
                .dossierTemplateId(annotateRequest.getDossierTemplateId())
+                        .colors(annotateRequest.getColors())
+                        .types(annotateRequest.getTypes())
                .build());
        var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());

@@ -0,0 +1,85 @@
+package com.iqser.red.service.redaction.v1.server.stringmatching;
+
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.SneakyThrows;
+import org.ahocorasick.trie.Trie;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+@RunWith(SpringRunner.class)
+public class StringMatchingPerformanceTest {
+
+    @Test
+    @SneakyThrows
+    public void testStringPerformance() {
+
+        String text = IOUtils.toString(new ClassPathResource("stringmatching/hamlet.txt").getInputStream()).toLowerCase();
+        Set<String> dictionary = IOUtils.readLines(new ClassPathResource("stringmatching/names.txt").getInputStream())
+                .stream()
+                .map(String::toLowerCase).collect(Collectors.toSet());
+
+        System.out.println("Loaded text has a length of " + text.length() + " symbols");
+        System.out.println("Dictionary has " + dictionary.size() + " entries");
+
+        var patterns = dictionary.stream()
+                .map(p -> Pattern.compile(Pattern.quote(p))).collect(Collectors.toList());
+        var trie = Trie.builder().ignoreCase().addKeywords(dictionary).build();
+
+
+        // 1. Naive approach
+        long t1 = System.currentTimeMillis();
+        var naiveIndexes = new HashSet<Index>();
+        for (var entry : dictionary) {
+            var startIndex = 0;
+            do {
+                startIndex = text.indexOf(entry, startIndex + 1);
+                if (startIndex != -1) {
+                    naiveIndexes.add(new Index(startIndex, startIndex + entry.length()));
+                }
+            } while (startIndex != -1);
+        }
+        long t2 = System.currentTimeMillis();
+        System.out.println("Naive approach found " + naiveIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // 2. Boyer Moore
+        t1 = System.currentTimeMillis();
+        var boyerMooreIndexes = new HashSet<Index>();
+        for (var pattern : patterns) {
+            boyerMooreIndexes.addAll(pattern.matcher(text).results().map(r -> new Index(r.start(), r.end())).collect(Collectors.toList()));
+        }
+        t2 = System.currentTimeMillis();
+        System.out.println("Boyer Moore found " + boyerMooreIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // 3. Aho Corasick
+        t1 = System.currentTimeMillis();
+        var result = trie.parseText(text);
+        var ahoCorasickIndexes = result.stream().map(r -> new Index(r.getStart(), r.getEnd() + 1)).collect(Collectors.toSet());
+        t2 = System.currentTimeMillis();
+        System.out.println("Aho Corasick found " + ahoCorasickIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // Assert that all algorithms are equal
+        assertThat(naiveIndexes).isEqualTo(boyerMooreIndexes).isEqualTo(ahoCorasickIndexes);
+
+    }
+
+    @AllArgsConstructor
+    @EqualsAndHashCode(of = {"start", "end"})
+    public static class Index {
+        int start;
+        int end;
+    }
+}
@@ -0,0 +1,13 @@
+{
+  "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+  "defaultColor": "#9398a0",
+  "requestAdd": "#04b093",
+  "requestRemove": "#04b093",
+  "notRedacted": "#c498fa",
+  "analysisColor": "#dd4d50",
+  "updatedColor": "#fdbd00",
+  "dictionaryRequestColor": "#5b97db",
+  "manualRedactionColor": "#9398a0",
+  "previewColor": "#9398a0",
+  "ignoredHintColor": "#e7d4ff"
+}
@@ -0,0 +1,223 @@
+[
+  {
+    "id": "CBI_address:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "CBI_address",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 140,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "All site names and addresses, and location (e.g. Syngenta, Monthey, GPS Co-ordinates, Mr Smith of … providing the…). Except addresses in published literature and the applicant address.",
+    "addToDictionaryAction": true,
+    "label": "CBI Address",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "CBI_author:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "CBI_author",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 130,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "All authors named in the study documentation. Except names in published literature.",
+    "addToDictionaryAction": true,
+    "label": "CBI Author",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "PII:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "PII",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 150,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Not authors but listed in the document: Names, signatures, telephone, email etc.; e.g. Reg Manager, QA Manager",
+    "addToDictionaryAction": true,
+    "label": "PII",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "formula:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "formula",
+    "hexColor": "#036ffc",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1002,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure formula colors.",
+    "addToDictionaryAction": false,
+    "label": "Formula",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "isHint_only:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "isHint_only",
+    "hexColor": "#fa98f7",
+    "recommendationHexColor": "#8df06c",
+    "rank": 50,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Entries of this dictionary will be highlighted only",
+    "addToDictionaryAction": false,
+    "label": "isHint Only",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "image:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "image",
+    "hexColor": "#bdd6ff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 999,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure image colors.",
+    "addToDictionaryAction": false,
+    "label": "Image",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "logo:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "logo",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1001,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure logo colors.",
+    "addToDictionaryAction": false,
+    "label": "Logo",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "must_redact:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "must_redact",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 100,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Entries of this dictionary get redacted wherever found.",
+    "addToDictionaryAction": false,
+    "label": "Must Redact",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "ocr:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "ocr",
+    "hexColor": "#bdd6ff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1000,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure ocr colors.",
+    "addToDictionaryAction": false,
+    "label": "Ocr",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "signature:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "signature",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1003,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure signature colors.",
+    "addToDictionaryAction": false,
+    "label": "Signature",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "imported_redaction:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "imported_redaction",
+    "hexColor": "#f0f0c0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 9999,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Redaction Annotations that were imported from documents",
+    "addToDictionaryAction": false,
+    "label": "Imported Redaction",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": true
+  },
+  {
+    "id": "published_information:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "published_information",
+    "hexColor": "#85ebff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 70,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Manual managed list of public journals and papers that need no redaction",
+    "addToDictionaryAction": true,
+    "label": "Published Information",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "dossier_redaction:31039447-9040-4376-9ca7-614e56b284b9:5dfb2724-74a4-4a1a-a1eb-165e7943ffcd",
+    "type": "dossier_redaction",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": null,
+    "rank": 1500,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Entries in this dictionary will only be redacted in this dossier",
+    "addToDictionaryAction": false,
+    "label": "Dossier Redaction",
+    "hasDictionary": true,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  }
+]
@@ -86,3 +86,4 @@ Toxicol Sci
 Toxicol Sci.
 Toxicol Sci. 1
 Test Ignored Hint Published Information
+Workshop
Author	SHA1	Message	Date
Timo Bejan	77584c9a5a	Pull request #394 : RED-3800 String Performance matching test Merge in RED/redaction-service from RED-3800-string-performance-test to master * commit '21d717f0837c9cc9b2f5b251d372db88b25a7b6d': RED-3800 String Performance matching test RED-3800 String Performance matching test	2022-05-24 11:09:08 +02:00
Timo Bejan	21d717f083	RED-3800 String Performance matching test	2022-05-24 12:01:10 +03:00
Timo Bejan	c85ce25ed4	RED-3800 String Performance matching test	2022-05-24 11:58:34 +03:00
Philipp Schramm	f608d991a3	Pull request #393 : RED-3816: Implemented ThenAction to redact section with rectangle Merge in RED/redaction-service from RED-3816 to master * commit 'eb7137315208dadf3a679e21b05ec16d53c4ec43': RED-3816: Implemented ThenAction to redact section with rectangle	2022-05-24 09:00:07 +02:00
Ali Oezyetimoglu	e914b09d1e	Pull request #392 : RED-3674: Fixed force redactions overrides idRemoval with different id Merge in RED/redaction-service from RED-3674-rs1 to master * commit '87d64b6d1285eaf7cd5ee3c2fb899f92063bc70f': RED-3674: Fixed force redactions overrides idRemoval with different id	2022-05-23 18:48:10 +02:00
aoezyetimoglu	87d64b6d12	RED-3674: Fixed force redactions overrides idRemoval with different id	2022-05-23 17:38:23 +02:00
Philipp Schramm	eb71373152	Merge branch 'master' into RED-3816	2022-05-23 16:27:12 +02:00
Dominique Eiflaender	4aaef260d7	Pull request #390 : RED-4075: Add legalBasis if Rule Entity overrides Ai Recommendation Merge in RED/redaction-service from RED-4075 to master * commit 'c669f9a09a3e5e254d41715f65fef3b158873338': RED-4075: Add legalBasis if Rule Entity overrides Ai Recommendation	2022-05-23 14:05:38 +02:00
deiflaender	c669f9a09a	RED-4075: Add legalBasis if Rule Entity overrides Ai Recommendation	2022-05-23 13:54:27 +02:00