RSS-106: Fixed not working redactions over multi pages

Merge in RED/redaction-service from RSS-92 to master

* commit '58c3d15b78e6ddf310b7288b4b703477c0a1218d':
  RSS-92: Fixed excludeHeadlines on tables

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -89,23 +89,36 @@ public class PlanSpec {
 
         return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
                 .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
-                                .cleanCheckout(true)
-                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
-                                .location(Location.FILE)
-                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
-                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                .enabled(true), new InjectVariablesTask().description("Inject git Tag")
-                                .path("git.tag")
-                                .namespace("g")
-                                .scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", 
+                        new BambooKey("JOB1")).tasks(
+                                new CleanWorkingDirectoryTask()
+                                        .description("Clean working directory.")
+                                        .enabled(true), 
+                                new VcsCheckoutTask().description("Checkout Default Repository")
+                                        .cleanCheckout(true)
+                                        .checkoutItems(
+                                                new CheckoutItem()
+                                                        .defaultRepository()), 
+                                new ScriptTask().description("Build")
+                                        .location(Location.FILE)
+                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                        .argument(SERVICE_NAME), createJUnitParserTask()
+                                        .description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true), 
+                                new InjectVariablesTask().description("Inject git Tag")
+                                        .path("git.tag")
+                                        .namespace("g")
+                                        .scope(InjectVariablesScope.LOCAL), 
+                                new VcsTagTask().description("${bamboo.g.gitTag}")
+                                        .tagName("${bamboo.g.gitTag}")
+                                        .defaultRepository())
+                .dockerConfiguration(
+                        new DockerConfiguration()
+                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                 .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                 .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RED / " + SERVICE_NAME)
-
                 .triggers(new BitbucketServerTrigger())
                 .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
                         .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
@@ -117,18 +130,29 @@ public class PlanSpec {
 
         return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
                 .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
-                                .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
-                                .cleanCheckout(true)
-                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
-                                .location(Location.FILE)
-                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
-                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                .enabled(true))
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .stages(new Stage("Default Stage").jobs(
+                        new Job("Default Job", new BambooKey("JOB1")).tasks(
+                                new CleanWorkingDirectoryTask()
+                                        .description("Clean working directory.")
+                                        .enabled(true), 
+                                new VcsCheckoutTask()
+                                        .description("Checkout Default Repository")
+                                        .cleanCheckout(true)
+                                        .checkoutItems(
+                                new CheckoutItem()
+                                        .defaultRepository()),
+                                new ScriptTask()
+                                        .description("Build")
+                                        .location(Location.FILE)
+                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                        .argument(SERVICE_NAME + " verify"), createJUnitParserTask().description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true))
+                        .dockerConfiguration(
+                                new DockerConfiguration()
+                                        .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                        .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                        .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RED / " + SERVICE_NAME)
                 .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
                 .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());

bamboo-specs/src/main/resources/scripts/build-java.sh

@@ -2,6 +2,7 @@
 set -e
 
 SERVICE_NAME=$1
+MVN_TARGET=${2:-deploy}
 
 if [[ "$bamboo_planRepository_branchName" == "master" ]]
 then
@@ -46,7 +47,7 @@ mvn --no-transfer-progress \
 
 mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
     --no-transfer-progress \
-    clean deploy \
+    clean $MVN_TARGET \
     ${bamboo_maven_add_param} \
     -e \
     -DdeployAtEnd=true \

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -54,6 +54,8 @@ public class Entity implements ReasonHolder {
 
     private boolean resized;
 
+    private boolean skipRemoveEntitiesContainedInLarger;
+
 
     public Entity(String word, String type, boolean redaction, String redactionReason,
                   List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -225,7 +225,11 @@ public class SearchableText {
 
     public String getAsStringWithLinebreaksSorted() {
 
+        return getAsStringWithLinebreaksSorted(this.sequences);
+    }
 
+
+    public String getAsStringWithLinebreaksSorted(List<TextPositionSequence> sequences) {
         var sorted = sequences.stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
 
         StringBuilder sb = new StringBuilder();
@@ -248,6 +252,7 @@ public class SearchableText {
 
 
 
+
     public String getAsStringWithLinebreaks() {
 
         StringBuilder sb = new StringBuilder();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -534,7 +534,7 @@ public class Section {
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                               @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false);
     }
 
 
@@ -545,7 +545,18 @@ public class Section {
                               @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis, @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger, @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
     }
 
 
@@ -556,7 +567,7 @@ public class Section {
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                  @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false, false);
     }
 
 
@@ -566,7 +577,7 @@ public class Section {
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                  @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false);
     }
 
 
@@ -580,6 +591,8 @@ public class Section {
     }
 
 
+
+
     @ThenAction
     @SuppressWarnings("unused")
     public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@@ -834,7 +847,7 @@ public class Section {
     public void redactSectionText(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
                                   @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true);
+        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true, false);
     }
 
 
@@ -843,7 +856,7 @@ public class Section {
     public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                                  @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true);
+        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true, false);
     }
 
 
@@ -1054,12 +1067,12 @@ public class Section {
 
 
     private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason, String legalBasis,
-                               boolean redaction) {
+                               boolean redaction, boolean skipRemoveEntitiesContainedInLarger, boolean sortedResult) {
 
         String[] values = new String[1];
 
         if (start.isEmpty() && stop.isEmpty()) {
-            if (excludeHeadLine) {
+            if (excludeHeadLine && searchText.contains(headline)) {
                 values[0] = StringUtils.substringAfter(searchText, headline);
             } else {
                 values[0] = searchText;
@@ -1077,6 +1090,13 @@ public class Section {
                 if (StringUtils.isNotBlank(value)) {
 
                     Set<Entity> found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    found.forEach(f -> {
+                        f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger);
+                        if(sortedResult){
+                            f.setWord(searchableText.getAsStringWithLinebreaksSorted(f.getPositionSequences().stream().map(EntityPositionSequence::getSequences).flatMap(Collection::stream).collect(Collectors.toList())));
+                        }
+                    });
+
                     EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
 
                     if (redactEverywhere && !isLocal()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -69,9 +69,26 @@ public class EntitySearchUtils {
             List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord()
                     .trim(), dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());
 
+            Map<String, List<EntityPositionSequence>> multipartSequenceParts = new HashMap<>();
+            Iterator<EntityPositionSequence> itty = positionSequences.iterator();
+            while (itty.hasNext()){
+                var positionSequence = itty.next();
+                if(positionSequence.getId().contains("-")){
+                    multipartSequenceParts.computeIfAbsent(positionSequence.getId().split("-")[0], (x) -> new ArrayList<>()).add(positionSequence);
+                    itty.remove();
+                }
+            }
+
             for (int i = 0; i <= orderedEntities.size() - 1; i++) {
                 try {
-                    orderedEntities.get(i).setPositionSequences(List.of(positionSequences.get(i)));
+                    List<EntityPositionSequence> toAdd = new ArrayList<>();
+                    var currentSeq = positionSequences.get(i);
+                    toAdd.add(currentSeq);
+                    if(multipartSequenceParts.containsKey(currentSeq.getId())){
+                        toAdd.addAll(multipartSequenceParts.get(currentSeq.getId()));
+                    }
+                    orderedEntities.get(i).setPositionSequences(toAdd);
+
                 } catch (Exception e) {
                     log.warn("Mismatch between EntityPositionSequence and found Entity!");
                 }
@@ -150,6 +167,10 @@ public class EntitySearchUtils {
 //                    continue;
 //                }
 
+                if(inner.isSkipRemoveEntitiesContainedInLarger() || word.isSkipRemoveEntitiesContainedInLarger()){
+                    continue;
+                }
+
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                     if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -390,40 +390,4 @@ rule "101: Redact CAS numbers"
         Section(hasTableHeader("Sample #"))
     then
         section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "102: Extract title"
-    when
-       Section(sectionNumber <= 3 && !text.contains("AUTHOR"))
-    then
-        section.redactBetween("", "Final Report", "PII", 5, false, "Title found", "n-a");
-        section.redactBetween("", "FINAL REPORT", "PII", 5, false, "Title found", "n-a");
-        section.redactBetween("TITLE", "DATA REQUIREMENT", "PII", 5, false, "Title found", "n-a");
-        section.redactBetween("TITLE", "DATA REQUIREMENT", "PII", 5, false, "Title found", "n-a");
-     end
-
-rule "102-1: Extract title"
-    when
-       Section(sectionNumber <= 3 && text.contains("SPL"))
-    then
-        section.redactBetween("Laboratories", "SPL", "PII", 5, false, "Title found", "n-a");
-     end
-
-
-rule "30: Vehicle components"
-
-    when
-
-        Section(text.contains("vehicle"))
-
-    then
-
-        //section.redactLineAfter("Name:","vehicle",30,true,"Vehicle components found.","n-a");
-
-        section.redactLineAfterAcrossColumns("Name:","PII",30,false,"bla","n-a");
-
-        section.redactLineAfter("Vehicle:","vehicle",30,true,"Vehicle keyword found.","n-a");
-
-        section.redactLineAfter("The vehicle was","vehicle",30,true,"Vehicle section found.","n-a");
-
     end