RED-5139: Bugfix if false positives are added, for rule added entries

Merge in RED/redaction-service from RED-5151 to master

* commit '0e71613ffcce147b2ae659e47ad6ab1c6405f0e7':
  RED-5151: Do not retry messages on oom errors

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -89,23 +89,36 @@ public class PlanSpec {
 
         return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
                 .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
-                                .cleanCheckout(true)
-                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
-                                .location(Location.FILE)
-                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
-                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                .enabled(true), new InjectVariablesTask().description("Inject git Tag")
-                                .path("git.tag")
-                                .namespace("g")
-                                .scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", 
+                        new BambooKey("JOB1")).tasks(
+                                new CleanWorkingDirectoryTask()
+                                        .description("Clean working directory.")
+                                        .enabled(true), 
+                                new VcsCheckoutTask().description("Checkout Default Repository")
+                                        .cleanCheckout(true)
+                                        .checkoutItems(
+                                                new CheckoutItem()
+                                                        .defaultRepository()), 
+                                new ScriptTask().description("Build")
+                                        .location(Location.FILE)
+                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                        .argument(SERVICE_NAME), createJUnitParserTask()
+                                        .description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true), 
+                                new InjectVariablesTask().description("Inject git Tag")
+                                        .path("git.tag")
+                                        .namespace("g")
+                                        .scope(InjectVariablesScope.LOCAL), 
+                                new VcsTagTask().description("${bamboo.g.gitTag}")
+                                        .tagName("${bamboo.g.gitTag}")
+                                        .defaultRepository())
+                .dockerConfiguration(
+                        new DockerConfiguration()
+                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                 .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                 .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RED / " + SERVICE_NAME)
-
                 .triggers(new BitbucketServerTrigger())
                 .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
                         .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
@@ -117,18 +130,29 @@ public class PlanSpec {
 
         return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
                 .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
-                                .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
-                                .cleanCheckout(true)
-                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
-                                .location(Location.FILE)
-                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
-                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                .enabled(true))
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .stages(new Stage("Default Stage").jobs(
+                        new Job("Default Job", new BambooKey("JOB1")).tasks(
+                                new CleanWorkingDirectoryTask()
+                                        .description("Clean working directory.")
+                                        .enabled(true), 
+                                new VcsCheckoutTask()
+                                        .description("Checkout Default Repository")
+                                        .cleanCheckout(true)
+                                        .checkoutItems(
+                                new CheckoutItem()
+                                        .defaultRepository()),
+                                new ScriptTask()
+                                        .description("Build")
+                                        .location(Location.FILE)
+                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                        .argument(SERVICE_NAME + " verify"), createJUnitParserTask().description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true))
+                        .dockerConfiguration(
+                                new DockerConfiguration()
+                                        .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                        .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                        .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RED / " + SERVICE_NAME)
                 .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
                 .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());

bamboo-specs/src/main/resources/scripts/build-java.sh

@@ -2,6 +2,7 @@
 set -e
 
 SERVICE_NAME=$1
+MVN_TARGET=${2:-deploy}
 
 if [[ "$bamboo_planRepository_branchName" == "master" ]]
 then
@@ -46,7 +47,7 @@ mvn --no-transfer-progress \
 
 mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
     --no-transfer-progress \
-    clean deploy \
+    clean $MVN_TARGET \
     ${bamboo_maven_add_param} \
     -e \
     -DdeployAtEnd=true \

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.queue;
 
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
 
+import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
@@ -18,8 +19,8 @@ public class MessageReceiver {
 
     @RabbitHandler
     @RabbitListener(queues = REDACTION_QUEUE)
-    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
-        redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+    public void receiveAnalyzeRequest(Message message) {
+        redactionMessageReceiver.receiveAnalyzeRequest(message, false);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.queue;
 
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
 
+import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
@@ -19,9 +20,9 @@ public class PriorityMessageReceiver {
 
     @RabbitHandler
     @RabbitListener(queues = REDACTION_PRIORITY_QUEUE)
-    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+    public void receiveAnalyzeRequest(Message message) {
 
-        redactionMessageReceiver.receiveAnalyzeRequest(in, true);
+        redactionMessageReceiver.receiveAnalyzeRequest(message, true);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -2,6 +2,8 @@ package com.iqser.red.service.redaction.v1.server.queue;
 
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
 
+import org.springframework.amqp.AmqpRejectAndDontRequeueException;
+import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.stereotype.Service;
@@ -16,6 +18,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeServic
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
 
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -29,11 +32,24 @@ public class RedactionMessageReceiver {
     private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
 
 
-    public void receiveAnalyzeRequest(String in, boolean priority) throws JsonProcessingException {
+    @SneakyThrows
+    public void receiveAnalyzeRequest(Message message, boolean priority) {
+
+        var analyzeRequest = objectMapper.readValue(message.getBody(), AnalyzeRequest.class);
+
+        // This prevents from endless retries oom errors.
+        if(message.getMessageProperties().isRedelivered()){
+            throw new AmqpRejectAndDontRequeueException(String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        }
+
+        receiveAnalyzeRequest(analyzeRequest, priority);
+    }
+
+    @SneakyThrows
+    public void receiveAnalyzeRequest(AnalyzeRequest analyzeRequest, boolean priority) {
 
-        var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
         log.info("Processing priority: {} analyze request for file: {}", priority, analyzeRequest.getFileId());
-        AnalyzeResult result = null;
+        AnalyzeResult result;
 
         try {
             switch (analyzeRequest.getMessageType()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -54,6 +54,8 @@ public class Entity implements ReasonHolder {
 
     private boolean resized;
 
+    private boolean skipRemoveEntitiesContainedInLarger;
+
 
     public Entity(String word, String type, boolean redaction, String redactionReason,
                   List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -225,7 +225,11 @@ public class SearchableText {
 
     public String getAsStringWithLinebreaksSorted() {
 
+        return getAsStringWithLinebreaksSorted(this.sequences);
+    }
 
+
+    public String getAsStringWithLinebreaksSorted(List<TextPositionSequence> sequences) {
         var sorted = sequences.stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
 
         StringBuilder sb = new StringBuilder();
@@ -248,6 +252,7 @@ public class SearchableText {
 
 
 
+
     public String getAsStringWithLinebreaks() {
 
         StringBuilder sb = new StringBuilder();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -530,51 +530,132 @@ public class Section {
     @Deprecated
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
-                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                              @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false);
     }
 
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
-                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason,
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                              @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.BOOLEAN) boolean includeStart,
+                              @Argument(ArgumentType.BOOLEAN) boolean includeStop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+
+        redactBetween(start, stop, includeStart, includeStop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
+    }
+
+
+    @ThenAction
+    public void redactBetweenRegexes(@Argument(ArgumentType.STRING) String startPattern,
+                                     @Argument(ArgumentType.BOOLEAN) boolean startPatternCaseInsensitive,
+                                     @Argument(ArgumentType.INTEGER) int startGroup,
+                                     @Argument(ArgumentType.BOOLEAN) boolean includeStart,
+                                     @Argument(ArgumentType.STRING) String stopPattern,
+                                     @Argument(ArgumentType.BOOLEAN) boolean stopPatternCaseInsensitive,
+                                     @Argument(ArgumentType.INTEGER) int stopGroup,
+                                     @Argument(ArgumentType.BOOLEAN) boolean includeStop,
+                                     @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                     @Argument(ArgumentType.TYPE) String type,
+                                     @Argument(ArgumentType.STRING) String reason,
+                                     @Argument(ArgumentType.STRING) String legalBasis,
+                                     @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                                     @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+
+        String startValue = getFirstRexExMatch(searchText, startPattern, startPatternCaseInsensitive, startGroup);
+
+        if (startValue == null){
+            return;
+        }
+
+        String searchTextAfter = StringUtils.substringAfter(searchText, startValue);
+
+        String stopValue = getFirstRexExMatch(searchTextAfter, stopPattern, stopPatternCaseInsensitive, stopGroup);
+
+        if (startValue != null && stopValue != null) {
+            redactBetween(startValue, stopValue, includeStart, includeStop, type, ruleNumber, false, false, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
+        }
     }
 
 
     @Deprecated
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
-                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+    public void redactNotBetween(@Argument(ArgumentType.STRING) String start,
+                                 @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                 @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                  @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false, false);
     }
 
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
-                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                                 @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason) {
+    public void redactNotBetween(@Argument(ArgumentType.STRING) String start,
+                                 @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                 @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                 @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                                 @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false);
     }
 
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
-                                   @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                                   @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+    public void redactLinesBetween(@Argument(ArgumentType.STRING) String start,
+                                   @Argument(ArgumentType.STRING) String stop,
+                                   @Argument(ArgumentType.TYPE) String asType,
+                                   @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                   @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                   @Argument(ArgumentType.STRING) String reason,
+                                   @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
     }
@@ -582,8 +663,11 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
-                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+    public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start,
+                                      @Argument(ArgumentType.STRING) String stop,
+                                      @Argument(ArgumentType.TYPE) String asType,
+                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                      @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                       @Argument(ArgumentType.STRING) String reason) {
 
         redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
@@ -726,7 +810,8 @@ public class Section {
     @SuppressWarnings("unused")
     public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) {
 
-        entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.RECOMMENDATION));
+        entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType()
+                .equals(EntityType.RECOMMENDATION));
     }
 
 
@@ -831,25 +916,45 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactSectionText(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
+    public void redactSectionText(@Argument(ArgumentType.TYPE) String type,
+                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                  @Argument(ArgumentType.STRING) String reason,
                                   @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true);
+        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true, false);
     }
 
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                                 @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+    public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type,
+                                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                                 @Argument(ArgumentType.STRING) String reason,
+                                                 @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true);
+        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true, false);
+    }
+
+
+    @ThenAction
+    public void redactHeadline(@Argument(ArgumentType.TYPE) String type,
+                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                               @Argument(ArgumentType.STRING) String reason,
+                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        if (searchText.contains(headline)) {
+
+            Set<Entity> found = findEntities(headline.trim(), type, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+            EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+        }
     }
 
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
+    public void redactSection(@Argument(ArgumentType.TYPE) String type,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         for (SectionArea sectionArea : sectionAreas) {
@@ -882,8 +987,8 @@ public class Section {
     }
 
 
-    private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis,
-                                           boolean redaction) {
+    private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
+                                           int ruleNumber, String reason, String legalBasis, boolean redaction) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
         Matcher matcher = compiledPattern.matcher(searchText);
@@ -898,6 +1003,9 @@ public class Section {
     }
 
 
+
+
+
     private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
                                      int ruleNumber, String reason, String legalBasis, Engine engine,
                                      boolean asRecommendation) {
@@ -970,22 +1078,25 @@ public class Section {
             singleEntitySet.add(entity);
 
             EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
+            EntitySearchUtils.removeFalsePositives(singleEntitySet, searchText, dictionary.getType(type), new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY));
 
-            EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
+            if (!singleEntitySet.isEmpty()) {
+                EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary);
 
-            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+                EntitySearchUtils.removeEntitiesContainedInLarger(entities);
 
-            if (addAsRecommendations && !isLocal()) {
-                String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
-                Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
-                Matcher matcher = pattern.matcher(cleanedWord);
+                if (addAsRecommendations && !isLocal()) {
+                    String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
+                    Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
+                    Matcher matcher = pattern.matcher(cleanedWord);
 
-                while (matcher.find()) {
-                    String match = matcher.group().trim();
-                    if (match.length() >= 3) {
-                        localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
-                        String lastname = match.split(" ")[0];
-                        localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
+                    while (matcher.find()) {
+                        String match = matcher.group().trim();
+                        if (match.length() >= 3) {
+                            localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
+                            String lastname = match.split(" ")[0];
+                            localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
+                        }
                     }
                 }
             }
@@ -1013,10 +1124,8 @@ public class Section {
     }
 
 
-
-    public void redactLineAfterAcrossColumns(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
-                                 String legalBasis) {
-
+    public void redactLineAfterAcrossColumns(String start, String asType, int ruleNumber, boolean redactEverywhere,
+                                             String reason, String legalBasis) {
 
         String[] values = StringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n");
 
@@ -1035,7 +1144,6 @@ public class Section {
     }
 
 
-
     private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
                                String reason, String legalBasis, boolean redaction) {
 
@@ -1053,13 +1161,31 @@ public class Section {
     }
 
 
-    private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason, String legalBasis,
-                               boolean redaction) {
+    private String getFirstRexExMatch(String text, String pattern, boolean patternCaseInsensitive, int group) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+        Matcher matcher = compiledPattern.matcher(text);
+
+        while (matcher.find()) {
+            String match = matcher.group(group);
+            if (StringUtils.isNotBlank(match)) {
+                return match;
+            }
+        }
+
+        return null;
+    }
+
+
+    private void redactBetween(String start, String stop, boolean includeStart, boolean includeStop, String asType,
+                               int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason,
+                               String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger,
+                               boolean sortedResult) {
 
         String[] values = new String[1];
 
         if (start.isEmpty() && stop.isEmpty()) {
-            if (excludeHeadLine) {
+            if (excludeHeadLine && searchText.contains(headline)) {
                 values[0] = StringUtils.substringAfter(searchText, headline);
             } else {
                 values[0] = searchText;
@@ -1076,11 +1202,32 @@ public class Section {
             for (String value : values) {
                 if (StringUtils.isNotBlank(value)) {
 
-                    Set<Entity> found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    String searchString = value;
+
+                    if (!start.isEmpty() && includeStart) {
+                        searchString = start + searchString;
+                    }
+
+                    if (!stop.isEmpty() && includeStop) {
+                        searchString = searchString + stop;
+                    }
+
+                    Set<Entity> found = findEntities(searchString.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    found.forEach(f -> {
+                        f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger);
+                        if (sortedResult) {
+                            f.setWord(searchableText.getAsStringWithLinebreaksSorted(f.getPositionSequences()
+                                    .stream()
+                                    .map(EntityPositionSequence::getSequences)
+                                    .flatMap(Collection::stream)
+                                    .collect(Collectors.toList())));
+                        }
+                    });
+
                     EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
 
                     if (redactEverywhere && !isLocal()) {
-                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(searchString.trim());
                     }
                 }
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -20,6 +20,13 @@ public class EntitySearchUtils {
         return searchImplementation.atLeastOneMatches(sectionText);
     }
 
+
+    public void removeFalsePositives(Set<Entity> found, String inputString, DictionaryModel type, FindEntityDetails details){
+        Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
+        markFalsePositives(found, falsePositives);
+        found.removeIf(f -> f.isFalsePositive());
+    }
+
     public Set<Entity> findEntities(String inputString, SearchImplementation searchImplementation, DictionaryModel type, FindEntityDetails details) {
 
         Set<Entity> found = find(inputString, searchImplementation, details);
@@ -69,9 +76,26 @@ public class EntitySearchUtils {
             List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord()
                     .trim(), dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());
 
+            Map<String, List<EntityPositionSequence>> multipartSequenceParts = new HashMap<>();
+            Iterator<EntityPositionSequence> itty = positionSequences.iterator();
+            while (itty.hasNext()){
+                var positionSequence = itty.next();
+                if(positionSequence.getId().contains("-")){
+                    multipartSequenceParts.computeIfAbsent(positionSequence.getId().split("-")[0], (x) -> new ArrayList<>()).add(positionSequence);
+                    itty.remove();
+                }
+            }
+
             for (int i = 0; i <= orderedEntities.size() - 1; i++) {
                 try {
-                    orderedEntities.get(i).setPositionSequences(List.of(positionSequences.get(i)));
+                    List<EntityPositionSequence> toAdd = new ArrayList<>();
+                    var currentSeq = positionSequences.get(i);
+                    toAdd.add(currentSeq);
+                    if(multipartSequenceParts.containsKey(currentSeq.getId())){
+                        toAdd.addAll(multipartSequenceParts.get(currentSeq.getId()));
+                    }
+                    orderedEntities.get(i).setPositionSequences(toAdd);
+
                 } catch (Exception e) {
                     log.warn("Mismatch between EntityPositionSequence and found Entity!");
                 }
@@ -150,6 +174,10 @@ public class EntitySearchUtils {
 //                    continue;
 //                }
 
+                if((inner.isSkipRemoveEntitiesContainedInLarger() || word.isSkipRemoveEntitiesContainedInLarger()) && !inner.getType().equals(word.getType())){
+                    continue;
+                }
+
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                     if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java

@@ -64,8 +64,7 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
             log.info("No text file provided, Performing Structure analysis");
 
             ar.setMessageType(MessageType.STRUCTURE_ANALYSE);
-            String in = om.writeValueAsString(ar);
-            redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+            redactionMessageReceiver.receiveAnalyzeRequest(ar, false);
         }
 
 
@@ -75,15 +74,13 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
             log.info("No redaction log provided, Performing full analysis");
 
             ar.setMessageType(MessageType.ANALYSE);
-            String in = om.writeValueAsString(ar);
-            redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+            redactionMessageReceiver.receiveAnalyzeRequest(ar, false);
         }
 
 
         simulateIncrement(List.of("Desiree"), "PII", 3L);
         ar.setMessageType(MessageType.REANALYSE);
-        String in = om.writeValueAsString(ar);
-        redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+        redactionMessageReceiver.receiveAnalyzeRequest(ar, false);
 
 
 //        var redactionLog = redactionStorageService.getRedactionLog("dossierId","fileId");

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -390,40 +390,4 @@ rule "101: Redact CAS numbers"
         Section(hasTableHeader("Sample #"))
     then
         section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "102: Extract title"
-    when
-       Section(sectionNumber <= 3 && !text.contains("AUTHOR"))
-    then
-        section.redactBetween("", "Final Report", "PII", 5, false, "Title found", "n-a");
-        section.redactBetween("", "FINAL REPORT", "PII", 5, false, "Title found", "n-a");
-        section.redactBetween("TITLE", "DATA REQUIREMENT", "PII", 5, false, "Title found", "n-a");
-        section.redactBetween("TITLE", "DATA REQUIREMENT", "PII", 5, false, "Title found", "n-a");
-     end
-
-rule "102-1: Extract title"
-    when
-       Section(sectionNumber <= 3 && text.contains("SPL"))
-    then
-        section.redactBetween("Laboratories", "SPL", "PII", 5, false, "Title found", "n-a");
-     end
-
-
-rule "30: Vehicle components"
-
-    when
-
-        Section(text.contains("vehicle"))
-
-    then
-
-        //section.redactLineAfter("Name:","vehicle",30,true,"Vehicle components found.","n-a");
-
-        section.redactLineAfterAcrossColumns("Name:","PII",30,false,"bla","n-a");
-
-        section.redactLineAfter("Vehicle:","vehicle",30,true,"Vehicle keyword found.","n-a");
-
-        section.redactLineAfter("The vehicle was","vehicle",30,true,"Vehicle section found.","n-a");
-
     end