RED-6093: Prototype find entities in rules

*added improved string to text position mapping

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -16,7 +16,6 @@ import java.util.Set;
 @AllArgsConstructor
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity implements ReasonHolder {
-
     private String word;
     private String type;
     private boolean redaction;
@@ -26,6 +25,7 @@ public class Entity implements ReasonHolder {
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();
     private List<TextPositionSequence> targetSequences;
 
+
     @EqualsAndHashCode.Include
     private Integer start;
     @EqualsAndHashCode.Include
@@ -66,6 +66,7 @@ public class Entity implements ReasonHolder {
                   String headline,
                   int matchedRule,
                   int sectionNumber,
+                  int paragraphNumber,
                   String legalBasis,
                   boolean isDictionaryEntry,
                   String textBefore,
@@ -85,6 +86,7 @@ public class Entity implements ReasonHolder {
         this.headline = headline;
         this.matchedRule = matchedRule;
         this.sectionNumber = sectionNumber;
+        this.paragraphNumber = paragraphNumber;
         this.legalBasis = legalBasis;
         this.isDictionaryEntry = isDictionaryEntry;
         this.textBefore = textBefore;
@@ -104,6 +106,7 @@ public class Entity implements ReasonHolder {
                   Integer end,
                   String headline,
                   int sectionNumber,
+                  int paragraphNumber,
                   boolean isDictionaryEntry,
                   boolean isDossierDictionaryEntry,
                   Engine engine,
@@ -115,6 +118,7 @@ public class Entity implements ReasonHolder {
         this.end = end;
         this.headline = headline;
         this.sectionNumber = sectionNumber;
+        this.paragraphNumber = paragraphNumber;
         this.isDictionaryEntry = isDictionaryEntry;
         this.isDossierDictionaryEntry = isDossierDictionaryEntry;
         this.engines.add(engine);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java

@@ -1,61 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString;
+import lombok.AccessLevel;
-
+import lombok.Builder;
-import java.util.List;
-
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
 import lombok.Getter;
+import lombok.experimental.FieldDefaults;
 
 @Getter
+@Builder
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class Paragraph {
 
-    private final String searchText;
+    SearchTextWithTextPositionModel searchTextToTextPosition;
-    private final List<RedTextPosition> textPositions;
+    int sectionNumber;
-    private final int[] searchTextToPositions;
+    int paragraphNumber;
-    private final int sectionNumber;
-
-    private final int start;
-    private final int end;
-
-
-    public Paragraph(TextBlock textBlock, int sectionNumber, int start, int end) {
-
-        textPositions = textBlock.getSequences().stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
-        this.searchTextToPositions = new int[textPositions.size()];
-        this.searchText = buildString(textBlock.getSequences());
-        this.start = start;
-        this.end = end;
-        this.sectionNumber = sectionNumber;
-    }
-
-    public boolean containsEntity(Entity entity) {
-        return entity.getStart() >= start && entity.getEnd() <= end;
-    }
-
-    private String buildSearchText() {
-
-        StringBuilder searchTextBuilder = new StringBuilder();
-        int numberOfNormalCharacters = 0;
-
-        for (int i = 0; i < textPositions.size(); ++i) {
-            if (isNormalText(textPositions.get(i).getUnicode())) {
-                searchTextBuilder.append(textPositions.get(i).getUnicode());
-                this.searchTextToPositions[numberOfNormalCharacters] = i;
-                ++numberOfNormalCharacters;
-            }
-        }
-        return searchTextBuilder.toString();
-    }
-
-
-    private boolean isNormalText(String unicode) {
-
-        return !unicode.equals("\n") && //
-                !unicode.equals("");
-
-    }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java

@@ -0,0 +1,22 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.List;
+import java.util.Map;
+
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Builder
+@Getter
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class SearchTextWithTextPositionModel {
+
+    String searchText;
+    Map<Integer, Integer> stringCoordsToTextPositionCoords;
+    List<Integer> lineBreaksStringCoords;
+    List<TextPositionSequence> textPositionSequences;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -1,17 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+
 import lombok.Getter;
 
 public class SearchableText {
@@ -184,30 +185,13 @@ public class SearchableText {
         return stringRepresentation;
     }
 
-
-//    public List<String> getParagraphStrings(){
-//
-//    }
-
-
     public static String buildString(List<TextPositionSequence> sequences) {
 
         StringBuilder sb = new StringBuilder();
-
-        TextPositionSequence previous = null;
         for (TextPositionSequence word : sequences) {
-
-            if (previous != null) {
-                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
-                    sb.append('\n');
-                } else {
-                    sb.append(' ');
-                }
-            }
             sb.append(word.toString());
-            previous = word;
+            sb.append(' ');
         }
-
         return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -1363,6 +1363,7 @@ public class Section {
                     value.getRowSpanStart() + word.length(),
                     headline,
                     sectionNumber,
+                    -1,
                     false,
                     false,
                     Engine.RULE,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -4,7 +4,7 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
-import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -22,6 +22,8 @@ import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
+import org.kie.api.runtime.rule.QueryResults;
+import org.kie.api.runtime.rule.QueryResultsRow;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@@ -88,19 +90,29 @@ public class DroolsExecutionService {
 
 
     @Timed("redactmanager_executeRules")
-    public List<Section> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
+    public List<Entity> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
 
-        Set<Entity> entities = new HashSet<>();
         KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("dictionary", dictionary);
         sections.forEach(kieSession::insert);
         paragraphs.forEach(kieSession::insert);
         kieSession.fireAllRules();
+        List<Entity> entities = getEntities(kieSession);
         kieSession.dispose();
 
-        return sections;
+        return entities;
 
     }
 
+    public List<Entity> getEntities(KieSession ks) {
+        List<Entity> entities = new LinkedList<>();
+        QueryResults entitiesResult = ks.getQueryResults("getEntities");
+        for (QueryResultsRow resultsRow : entitiesResult) {
+            entities.add((Entity) resultsRow.get("$result"));
+        }
+        return entities;
+    }
+
 
     public KieContainer updateRules(String dossierTemplateId) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java

@@ -0,0 +1,81 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
+
+@Service
+public class SearchTextWithTextPositionFactory {
+
+    public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
+
+        Map<Integer, Integer> stringIdxToPositionIdx = new HashMap<>();
+        List<Integer> lineBreaksStringIdx = new LinkedList<>();
+        StringBuilder sb = new StringBuilder();
+
+        int stringIdx = 0;
+        int positionIdx = 0;
+
+        String currentUnicode;
+        String previousUnicode = " ";
+
+        for (TextPositionSequence word : sequences) {
+            for (int i = 0; i < word.getTextPositions().size(); ++i) {
+
+                currentUnicode = word.getTextPositions().get(i).getUnicode();
+
+                if (isLineBreak(currentUnicode)) {
+                    lineBreaksStringIdx.add(stringIdx);
+                } else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && //
+                        !isHyphenLinebreak(currentUnicode)) {
+
+                    sb.append(currentUnicode);
+                    stringIdxToPositionIdx.put(stringIdx, positionIdx);
+                    ++stringIdx;
+                }
+
+                previousUnicode = currentUnicode;
+                ++positionIdx;
+            }
+
+            previousUnicode = " ";
+            sb.append(previousUnicode);
+            stringIdxToPositionIdx.put(stringIdx, positionIdx);
+            ++stringIdx;
+        }
+
+        return SearchTextWithTextPositionModel.builder()
+                .searchText(sb.toString().stripTrailing())
+                .lineBreaksStringCoords(lineBreaksStringIdx)
+                .stringCoordsToTextPositionCoords(stringIdxToPositionIdx)
+                .textPositionSequences(sequences)
+                .build();
+    }
+
+
+    private boolean isLineBreak(String currentUnicode) {
+
+        return Objects.equals(currentUnicode, "\n");
+    }
+
+
+    private boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) {
+
+        return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " ");
+    }
+
+
+    private boolean isHyphenLinebreak(String unicodeCharacter) {
+
+        return unicodeCharacter.matches("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R");
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java

@@ -1,10 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-import org.ahocorasick.trie.Trie;
-
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -12,6 +7,10 @@ import java.util.Locale;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.ahocorasick.trie.Trie;
+
+import lombok.Data;
+
 @Data
 public class SearchImplementation {
 
@@ -94,13 +93,7 @@ public class SearchImplementation {
         }
     }
 
-
+    public record MatchPosition(int startIndex, int endIndex) {
-    @Data
-    @AllArgsConstructor
-    public static class MatchPosition {
-
-        private int startIndex;
-        private int endIndex;
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java

@@ -17,8 +17,8 @@ public class EntitySearchUtilsTest {
     public void testNestedEntitiesRemoval() {
 
         Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         entities.add(nested);
         entities.add(nesting);
         EntitySearchUtils.removeEntitiesContainedInLarger(entities);
@@ -40,14 +40,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -73,14 +73,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -105,14 +105,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -137,15 +137,15 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
         foundEntities.add(foundEntities3);
@@ -171,16 +171,16 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntitiesOverlap1);
         foundEntities.add(foundEntitiesOverlap2);
         foundEntities.add(foundEntitiesSubset1);

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -1,390 +1,67 @@
 package drools
 
-import com.iqser.red.service.redaction.v1.server.redaction.model.*
+import static java.lang.String.format;
 
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
 
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import java.util.Set;
+
+global Dictionary dictionary
+
+query "getEntities"
+    $result: Entity()
+end
 
 // --------------------------------------- AI rules -------------------------------------------------------------------
-rule "-1: find entities from dictionary"
+rule "0: find entities from entry dictionary"
     salience 100
-    no-loop true
     when
-        section: Section(!text.isEmpty())
+        $section: Section()
+        $paragraph: Paragraph(sectionNumber == $section.sectionNumber)
+        $entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition())
     then
-        section.findDictionaryEntities();
+        List<EntityPositionSequence> eps = new LinkedList<>();
-        update(section);
+        eps.add($entityPositionSequence);
-     end
+        Set<Engine> engineSet = new HashSet<>();
-
+        engineSet.add(Engine.DICTIONARY);
-rule "0: Add CBI_author from ai"
+        Set<Entity> empty = new HashSet<>();
-    when
+        insert(new Entity("CBI_author",
-        section: Section(aiMatchesType("CBI_author"))
+               "custom",
-    then
+               true,
-        section.addAiEntities("CBI_author", "CBI_author");
+               format("Found in Dictionary: %s", "custom"),
-    end
+               eps,
-
+               $section.getHeadline(),
-rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
+               0,
-    when
+               $section.getSectionNumber(),
-        section: Section(aiMatchesType("ORG"))
+               $paragraph.getParagraphNumber(),
-    then
+               "",
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+               true,
-    end
+               "",
-
+               "",
-rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
+               -1,
-    when
+               -1,
-        section: Section(aiMatchesType("STREET"))
+               false,
-    then
+               engineSet,
-        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+               empty,
-    end
+               EntityType.ENTITY
-
+               ));
-rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
-    when
-        section: Section(aiMatchesType("CITY"))
-    then
-        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
-    end
-
-
-// --------------------------------------- CBI rules -------------------------------------------------------------------
-
-rule "1: Redact CBI Authors (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
-    then
-        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "2: Redact CBI Authors (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
-    then
-        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "3: Redact not CBI Address (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
-    then
-        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
-        section.ignoreRecommendations("CBI_address");
-    end
-
-rule "4: Redact CBI Address (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
-    then
-        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "5: Do not redact genitive CBI_author"
-    when
-        section: Section(matchesType("CBI_author"))
-    then
-        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
-    end
-
-
-rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
-    then
-        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
-    then
-        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
-    then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
-    then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "16: Add recommendation for Addresses in Test Organism sections"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
-    then
-        section.recommendLineAfter("Source:", "CBI_address");
-    end
-
-rule "17: Add recommendation for Addresses in Test Animals sections"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
-    then
-        section.recommendLineAfter("Source", "CBI_address");
     end
 
 /*
-
+rule "44: Don't redact Author if contained in paragraph with published information"
-rule "18: Do not redact Names and Addresses if Published Information found"
-    when
-        section: Section(matchesType("published_information"))
-    then
-        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
-        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
-    end
-
-*/
-
-// --------------------------------------- PII rules -------------------------------------------------------------------
-
-
-rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
-    then
-        section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
-    then
-        section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "21: Redact Emails by RegEx (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
-    then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "22: Redact Emails by RegEx (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
-    then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "23: Redact contact information (Non vertebrate study)"
-    when
-       section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
-            || text.contains("Contact:")
-            || text.contains("Alternative contact:")
-            || (text.contains("No:") && text.contains("Fax"))
-            || (text.contains("Contact:") && text.contains("Tel.:"))
-            || text.contains("European contact:")
-        ))
-    then
-        section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("European contact:",  "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-     end
-
-rule "24: Redact contact information (Vertebrate study)"
-    when
-       section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
-            || text.contains("Contact:")
-            || text.contains("Alternative contact:")
-            || (text.contains("No:") && text.contains("Fax"))
-            || (text.contains("Contact:") && text.contains("Tel.:"))
-            || text.contains("European contact:")
-        ))
-    then
-        section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("European contact:",  "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-     end
-
-rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
-               text.contains("Contact")
-            || text.contains("Telephone")
-            || text.contains("Phone")
-            || text.contains("Fax")
-            || text.contains("Tel")
-            || text.contains("Ter")
-            || text.contains("Mobile")
-            || text.contains("Fel")
-            || text.contains("Fer")
-        ))
-    then
-        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
-               text.contains("Contact")
-            || text.contains("Telephone")
-            || text.contains("Phone")
-            || text.contains("Fax")
-            || text.contains("Tel")
-            || text.contains("Ter")
-            || text.contains("Mobile")
-            || text.contains("Fel")
-            || text.contains("Fer")
-        ))
-    then
-        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "27: Redact AUTHOR(S) (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("COMPLETION DATE:")
-            && !searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "28: Redact AUTHOR(S) (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("COMPLETION DATE:")
-            && !searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "29: Redact AUTHOR(S) (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "30: Redact AUTHOR(S) (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("PERFORMING LABORATORY:")
-        )
-    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
-    end
-
-rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("PERFORMING LABORATORY:"))
-    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "33: Redact study director abbreviation"
-    when
-        section: Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
-    then
-        section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-     end
-
-
-// --------------------------------------- other rules -------------------------------------------------------------------
-
-rule "34: Purity Hint"
-    when
-        section: Section(searchText.toLowerCase().contains("purity"))
-    then
-	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
-    end
-
-
-
-rule "35: Redact signatures (Non vertebrate study)"
-    when
-        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
-    then
-        section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "36: Redact signatures (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
-    then
-        section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "43: Redact Logos (Vertebrate study)"
-    when
-        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
-    then
-        section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "44: Redact Author if contained in paragraph with 'unpublished'"
     when
         $section: Section()
-        $entity: Entity(type == "CBI_author") from $section.getEntities()
+        $paragraph: Paragraph($section.sectionNumber == sectionNumber)
-        $publishedEntity: Entity(type == "published_information") from $section.getEntities()
-        $paragraph: Paragraph(sectionNumber == $section.sectionNumber)
-    then
-        if ($paragraph.containsEntity($entity) && $paragraph.containsEntity($publishedEntity) ) {
 
-            $entity.setRedaction(false);
+        $authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber())
-        }
+        $publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber())
+    then
+        $authorEntity.setRedaction(false);
     end
+*/