Revert idRemoval fix, removed duplicate code

Merge in RED/redaction-service from annotation-fix to master

* commit 'e3a960d0861bdfa96b90fe406eb825573007530e':
  idRemoval fix

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -144,7 +144,7 @@ public class Section {
     public void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis) {
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
-        Matcher matcher = compiledPattern.matcher(text);
+        Matcher matcher = compiledPattern.matcher(searchText);
 
         while (matcher.find()) {
             String match = matcher.group(group);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -7,7 +7,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -290,47 +289,13 @@ public class EntityRedactionService {
         String lowercaseInputString = searchableString.toLowerCase();
         for (DictionaryModel model : dictionary.getDictionaryModels()) {
             if (model.isCaseInsensitive()) {
-                found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
             } else {
-                found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
             }
         }
 
         return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
-
-    }
-
-
-    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                             boolean local) {
-
-        Set<Entity> found = new HashSet<>();
-
-        for (String value : values) {
-
-            if (value.trim().length() <= 2) {
-                continue;
-            }
-
-            int startIndex;
-            int stopIndex = 0;
-            do {
-                startIndex = inputString.indexOf(value, stopIndex);
-                stopIndex = startIndex + value.length();
-
-                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
-                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local));
-                }
-            } while (startIndex > -1);
-        }
-        return found;
-    }
-
-
-    private boolean isSeparator(char c) {
-
-        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -29,15 +29,17 @@ public class EntitySearchUtils {
 
         for (String value : values) {
 
-            if (value.trim().length() <= 2) {
+            String cleanValue = value.trim();
+
+            if (cleanValue.length() <= 2) {
                 continue;
             }
 
             int startIndex;
             int stopIndex = 0;
             do {
-                startIndex = inputString.indexOf(value, stopIndex);
-                stopIndex = startIndex + value.length();
+                startIndex = inputString.indexOf(cleanValue, stopIndex);
+                stopIndex = startIndex + cleanValue.length();
 
                 if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
                         .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
@@ -70,7 +72,7 @@ public class EntitySearchUtils {
                     .sorted(Comparator.comparing(Entity::getStart))
                     .collect(Collectors.toList());
             Entity firstEntity = orderedEntities.get(0);
-            List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord(), dictionary.isCaseInsensitiveDictionary(firstEntity
+            List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord().trim(), dictionary.isCaseInsensitiveDictionary(firstEntity
                     .getType()), firstEntity.getTargetSequences());
 
             for (int i = 0; i <= orderedEntities.size() - 1; i++) {

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -138,7 +138,7 @@ rule "12: Add recommendation for et al. author"
     when
         Section(searchText.contains("et al."))
     then
-		section.addRecommendationByRegEx("([^\\s(]*?( \\w\\.?)?) et al\\.?", false, 1, "CBI_author");
+		section.addRecommendationByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]\\.?)?( [A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author");
     end
 
 // --------------------------------------- PII rules -------------------------------------------------------------------