Pull request #472 : RSS-105: Use string after match of first regEx for second regex in redactBetweenRegexes

Merge in RED/redaction-service from RSS-105 to master * commit '5503dbfffc10ef9f51da7c37cb66e933d747a874': RSS-105: Use string after match of first regEx for second regex in redactBetweenRegexes
RSS-105: Use string after match of first regEx for second regex in redactBetweenRegexes
2022-09-15 13:08:42 +02:00 · 2022-09-15 13:03:31 +02:00 · 2022-09-15 11:47:22 +02:00 · 2022-09-15 11:41:50 +02:00 · 2022-09-13 12:12:35 +02:00 · 2022-09-13 12:09:09 +02:00
39 changed files with 372 additions and 46 deletions
@@ -89,23 +89,36 @@ public class PlanSpec {

        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
                .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
-                                .cleanCheckout(true)
-                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
-                                .location(Location.FILE)
-                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
-                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                .enabled(true), new InjectVariablesTask().description("Inject git Tag")
-                                .path("git.tag")
-                                .namespace("g")
-                                .scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", 
+                        new BambooKey("JOB1")).tasks(
+                                new CleanWorkingDirectoryTask()
+                                        .description("Clean working directory.")
+                                        .enabled(true), 
+                                new VcsCheckoutTask().description("Checkout Default Repository")
+                                        .cleanCheckout(true)
+                                        .checkoutItems(
+                                                new CheckoutItem()
+                                                        .defaultRepository()), 
+                                new ScriptTask().description("Build")
+                                        .location(Location.FILE)
+                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                        .argument(SERVICE_NAME), createJUnitParserTask()
+                                        .description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true), 
+                                new InjectVariablesTask().description("Inject git Tag")
+                                        .path("git.tag")
+                                        .namespace("g")
+                                        .scope(InjectVariablesScope.LOCAL), 
+                                new VcsTagTask().description("${bamboo.g.gitTag}")
+                                        .tagName("${bamboo.g.gitTag}")
+                                        .defaultRepository())
+                .dockerConfiguration(
+                        new DockerConfiguration()
+                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                .linkedRepositories("RED / " + SERVICE_NAME)
-
                .triggers(new BitbucketServerTrigger())
                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
@@ -117,18 +130,29 @@ public class PlanSpec {

        return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
                .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
-                                .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
-                                .cleanCheckout(true)
-                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
-                                .location(Location.FILE)
-                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
-                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                .enabled(true))
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .stages(new Stage("Default Stage").jobs(
+                        new Job("Default Job", new BambooKey("JOB1")).tasks(
+                                new CleanWorkingDirectoryTask()
+                                        .description("Clean working directory.")
+                                        .enabled(true), 
+                                new VcsCheckoutTask()
+                                        .description("Checkout Default Repository")
+                                        .cleanCheckout(true)
+                                        .checkoutItems(
+                                new CheckoutItem()
+                                        .defaultRepository()),
+                                new ScriptTask()
+                                        .description("Build")
+                                        .location(Location.FILE)
+                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                        .argument(SERVICE_NAME + " verify"), createJUnitParserTask().description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true))
+                        .dockerConfiguration(
+                                new DockerConfiguration()
+                                        .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                        .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                        .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                .linkedRepositories("RED / " + SERVICE_NAME)
                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
                .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
@@ -2,6 +2,7 @@
 set -e

 SERVICE_NAME=$1
+MVN_TARGET=${2:-deploy}

 if [[ "$bamboo_planRepository_branchName" == "master" ]]
 then
@@ -46,7 +47,7 @@ mvn --no-transfer-progress \

 mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
    --no-transfer-progress \
-    clean deploy \
+    clean $MVN_TARGET \
    ${bamboo_maven_add_param} \
    -e \
    -DdeployAtEnd=true \
@@ -54,6 +54,8 @@ public class Entity implements ReasonHolder {

    private boolean resized;

+    private boolean skipRemoveEntitiesContainedInLarger;
+

    public Entity(String word, String type, boolean redaction, String redactionReason,
                  List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber,
@@ -9,7 +9,11 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati

 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Getter;

 public class SearchableText {

@@ -17,6 +21,7 @@ public class SearchableText {
    @JsonIgnore
    @JsonAttribute(ignore = true)
    private transient String stringRepresentation;
+    @Getter
    private final List<TextPositionSequence> sequences = new ArrayList<>();


@@ -218,6 +223,36 @@ public class SearchableText {
    }


+    public String getAsStringWithLinebreaksSorted() {
+
+        return getAsStringWithLinebreaksSorted(this.sequences);
+    }
+
+
+    public String getAsStringWithLinebreaksSorted(List<TextPositionSequence> sequences) {
+        var sorted = sequences.stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
+
+        StringBuilder sb = new StringBuilder();
+
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : sorted) {
+
+            if (previous != null) {
+                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                    sb.append('\n');
+                } else {
+                    sb.append(' ');
+                }
+            }
+            sb.append(word.toString());
+            previous = word;
+        }
+        return sb.append("\n").toString();
+    }
+
+
+
+
    public String getAsStringWithLinebreaks() {

        StringBuilder sb = new StringBuilder();
@@ -1,5 +1,23 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
 import com.iqser.red.service.redaction.v1.model.Engine;
@@ -10,19 +28,10 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+
 import lombok.Builder;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;

@Data
@Slf4j
@@ -518,6 +527,7 @@ public class Section {
    }


+    @Deprecated
    @ThenAction
    @SuppressWarnings("unused")
    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
@@ -527,7 +537,100 @@ public class Section {
                              @Argument(ArgumentType.STRING) String reason,
                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {

-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.BOOLEAN) boolean includeStart,
+                              @Argument(ArgumentType.BOOLEAN) boolean includeStop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+
+        redactBetween(start, stop, includeStart, includeStop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
+    }
+
+
+    @ThenAction
+    public void redactBetweenRegexes(@Argument(ArgumentType.STRING) String startPattern,
+                                     @Argument(ArgumentType.BOOLEAN) boolean startPatternCaseInsensitive,
+                                     @Argument(ArgumentType.INTEGER) int startGroup,
+                                     @Argument(ArgumentType.BOOLEAN) boolean includeStart,
+                                     @Argument(ArgumentType.STRING) String stopPattern,
+                                     @Argument(ArgumentType.BOOLEAN) boolean stopPatternCaseInsensitive,
+                                     @Argument(ArgumentType.INTEGER) int stopGroup,
+                                     @Argument(ArgumentType.BOOLEAN) boolean includeStop,
+                                     @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                     @Argument(ArgumentType.TYPE) String type,
+                                     @Argument(ArgumentType.STRING) String reason,
+                                     @Argument(ArgumentType.STRING) String legalBasis,
+                                     @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                                     @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+
+        String startValue = getFirstRexExMatch(searchText, startPattern, startPatternCaseInsensitive, startGroup);
+
+        if (startValue == null){
+            return;
+        }
+
+        String searchTextAfter = StringUtils.substringAfter(searchText, startValue);
+
+        String stopValue = getFirstRexExMatch(searchTextAfter, stopPattern, stopPatternCaseInsensitive, stopGroup);
+
+        if (startValue != null && stopValue != null) {
+            redactBetween(startValue, stopValue, includeStart, includeStop, type, ruleNumber, false, false, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
+        }
+    }
+
+
+    @Deprecated
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactNotBetween(@Argument(ArgumentType.STRING) String start,
+                                 @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                 @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                 @Argument(ArgumentType.STRING) String reason) {
+
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false, false);
    }


@@ -537,9 +640,10 @@ public class Section {
                                 @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                 @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                 @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
                                 @Argument(ArgumentType.STRING) String reason) {

-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false);
    }


@@ -810,6 +914,42 @@ public class Section {
    }


+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactSectionText(@Argument(ArgumentType.TYPE) String type,
+                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                  @Argument(ArgumentType.STRING) String reason,
+                                  @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type,
+                                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                                 @Argument(ArgumentType.STRING) String reason,
+                                                 @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true, false);
+    }
+
+
+    @ThenAction
+    public void redactHeadline(@Argument(ArgumentType.TYPE) String type,
+                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                               @Argument(ArgumentType.STRING) String reason,
+                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        if (searchText.contains(headline)) {
+
+            Set<Entity> found = findEntities(headline.trim(), type, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+            EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+        }
+    }
+
+
    @ThenAction
    @SuppressWarnings("unused")
    public void redactSection(@Argument(ArgumentType.TYPE) String type,
@@ -978,6 +1118,26 @@ public class Section {
    }


+    public void redactLineAfterAcrossColumns(String start, String asType, int ruleNumber, boolean redactEverywhere,
+                                             String reason, String legalBasis) {
+
+        String[] values = StringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n");
+
+        if (values != null) {
+            for (String value : values) {
+                if (StringUtils.isNotBlank(value)) {
+                    Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+
+                    if (redactEverywhere && !isLocal()) {
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
+                    }
+                }
+            }
+        }
+    }
+
+
    private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
                               String reason, String legalBasis, boolean redaction) {

@@ -995,20 +1155,73 @@ public class Section {
    }


-    private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
-                               String reason, String legalBasis, boolean redaction) {
+    private String getFirstRexExMatch(String text, String pattern, boolean patternCaseInsensitive, int group) {

-        String[] values = StringUtils.substringsBetween(searchText, start, stop);
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+        Matcher matcher = compiledPattern.matcher(text);
+
+        while (matcher.find()) {
+            String match = matcher.group(group);
+            if (StringUtils.isNotBlank(match)) {
+                return match;
+            }
+        }
+
+        return null;
+    }
+
+
+    private void redactBetween(String start, String stop, boolean includeStart, boolean includeStop, String asType,
+                               int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason,
+                               String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger,
+                               boolean sortedResult) {
+
+        String[] values = new String[1];
+
+        if (start.isEmpty() && stop.isEmpty()) {
+            if (excludeHeadLine && searchText.contains(headline)) {
+                values[0] = StringUtils.substringAfter(searchText, headline);
+            } else {
+                values[0] = searchText;
+            }
+        } else if (start.isEmpty() && searchText.contains(stop)) {
+            values[0] = StringUtils.substringBefore(searchText, stop);
+        } else if (stop.isEmpty() && searchText.contains(start)) {
+            values[0] = StringUtils.substringAfter(searchText, start);
+        } else {
+            values = StringUtils.substringsBetween(searchText, start, stop);
+        }

        if (values != null) {
            for (String value : values) {
                if (StringUtils.isNotBlank(value)) {

-                    Set<Entity> found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    String searchString = value;
+
+                    if (!start.isEmpty() && includeStart) {
+                        searchString = start + searchString;
+                    }
+
+                    if (!stop.isEmpty() && includeStop) {
+                        searchString = searchString + stop;
+                    }
+
+                    Set<Entity> found = findEntities(searchString.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    found.forEach(f -> {
+                        f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger);
+                        if (sortedResult) {
+                            f.setWord(searchableText.getAsStringWithLinebreaksSorted(f.getPositionSequences()
+                                    .stream()
+                                    .map(EntityPositionSequence::getSequences)
+                                    .flatMap(Collection::stream)
+                                    .collect(Collectors.toList())));
+                        }
+                    });
+
                    EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);

                    if (redactEverywhere && !isLocal()) {
-                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(searchString.trim());
                    }
                }
            }
@@ -69,9 +69,26 @@ public class EntitySearchUtils {
            List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord()
                    .trim(), dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());

+            Map<String, List<EntityPositionSequence>> multipartSequenceParts = new HashMap<>();
+            Iterator<EntityPositionSequence> itty = positionSequences.iterator();
+            while (itty.hasNext()){
+                var positionSequence = itty.next();
+                if(positionSequence.getId().contains("-")){
+                    multipartSequenceParts.computeIfAbsent(positionSequence.getId().split("-")[0], (x) -> new ArrayList<>()).add(positionSequence);
+                    itty.remove();
+                }
+            }
+
            for (int i = 0; i <= orderedEntities.size() - 1; i++) {
                try {
-                    orderedEntities.get(i).setPositionSequences(List.of(positionSequences.get(i)));
+                    List<EntityPositionSequence> toAdd = new ArrayList<>();
+                    var currentSeq = positionSequences.get(i);
+                    toAdd.add(currentSeq);
+                    if(multipartSequenceParts.containsKey(currentSeq.getId())){
+                        toAdd.addAll(multipartSequenceParts.get(currentSeq.getId()));
+                    }
+                    orderedEntities.get(i).setPositionSequences(toAdd);
+
                } catch (Exception e) {
                    log.warn("Mismatch between EntityPositionSequence and found Entity!");
                }
@@ -150,6 +167,10 @@ public class EntitySearchUtils {
 //                    continue;
 //                }

+                if(inner.isSkipRemoveEntitiesContainedInLarger() || word.isSkipRemoveEntitiesContainedInLarger()){
+                    continue;
+                }
+
                if (inner.getWord().length() < word.getWord()
                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
@@ -330,6 +330,36 @@ public class RedactionIntegrationTest {
    }


+
+    @Test
+    public void titleExtraction() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/32 - Emamectin Benzoate Technical - Acute Oral Toxicity - Mouse.pdf");
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+
+
+
    @Test
    @Ignore
    @SneakyThrows
@@ -1064,7 +1094,7 @@ public class RedactionIntegrationTest {
    public void classificationTest() throws IOException {

        System.out.println("classificationTest");
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");

        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
Author	SHA1	Message	Date
Dominique Eiflaender	d2f2cd975c	Pull request #472 : RSS-105: Use string after match of first regEx for second regex in redactBetweenRegexes Merge in RED/redaction-service from RSS-105 to master * commit '5503dbfffc10ef9f51da7c37cb66e933d747a874': RSS-105: Use string after match of first regEx for second regex in redactBetweenRegexes	2022-09-15 13:08:42 +02:00
deiflaender	5503dbfffc	RSS-105: Use string after match of first regEx for second regex in redactBetweenRegexes	2022-09-15 13:03:31 +02:00
Dominique Eiflaender	369710dbd4	Pull request #471 : RSS-109: Added rule redactBetweenRegexes Merge in RED/redaction-service from RSS-109 to master * commit 'b9b3e8c8f5f11fbce6d0ae85f8e079e6285b70e0': RSS-109: Added rule redactBetweenRegexes	2022-09-15 11:47:22 +02:00
deiflaender	b9b3e8c8f5	RSS-109: Added rule redactBetweenRegexes	2022-09-15 11:41:50 +02:00
Dominique Eiflaender	c478935b86	Pull request #470 : RSS-107: Added funktion to redact Headline Merge in RED/redaction-service from RSS-107 to master * commit 'd92757cda462d8af9ac852b88c867032c753a3b0': RSS-107: Added funktion to redact Headline	2022-09-13 12:12:35 +02:00
deiflaender	d92757cda4	RSS-107: Added funktion to redact Headline	2022-09-13 12:09:09 +02:00
Dominique Eiflaender	5828e19422	Pull request #469 : RSS-105: Added parameters to include/exclude start/stop in redactBetween rule Merge in RED/redaction-service from RSS-105 to master * commit 'b7bf84c323b1e730481dad16428db01227d3347e': RSS-105: Fixed checkstyle error RSS-105: Added parameters to include/exclude start/stop in redactBetween rule	2022-09-13 11:49:42 +02:00
deiflaender	b7bf84c323	RSS-105: Fixed checkstyle error	2022-09-13 11:44:28 +02:00
deiflaender	fe7e8a83df	RSS-105: Added parameters to include/exclude start/stop in redactBetween rule	2022-09-13 11:35:36 +02:00
Dominique Eiflaender	a71a7818c2	Pull request #468 : RSS-106: Fixed not working redactions over multi pages Merge in RED/redaction-service from RSS-106 to master * commit 'ee5da81299425452171370ab609a0172858c4787': RSS-106: Fixed not working redactions over multi pages	2022-09-13 08:57:27 +02:00
deiflaender	ee5da81299	RSS-106: Fixed not working redactions over multi pages	2022-09-12 17:40:16 +02:00
Dominique Eiflaender	e486711755	Pull request #467 : RSS-92: Fixed excludeHeadlines on tables Merge in RED/redaction-service from RSS-92 to master * commit '58c3d15b78e6ddf310b7288b4b703477c0a1218d': RSS-92: Fixed excludeHeadlines on tables	2022-09-12 13:45:19 +02:00
deiflaender	58c3d15b78	RSS-92: Fixed excludeHeadlines on tables	2022-09-12 12:58:15 +02:00
Christoph Schabert	ce15faf072	Pull request #466 : hotfix: remove push from nightly build Merge in RED/redaction-service from fixNightly to master * commit '430a08e611cb6ac99a31139f1a46bf36a66bb6ab': hotfix: remove push from nightly build	2022-09-12 11:29:26 +02:00
cschabert	430a08e611	hotfix: remove push from nightly build	2022-09-12 11:26:14 +02:00
Dominique Eiflaender	4cc40d8381	Pull request #465 : RSS-31: Allow to skip removeEntitiesContainedInLarger in redactBetween rule and added param to sort result by positions Merge in RED/redaction-service from RSS-31 to master * commit '317a8a9af9d9755befe66d82eef5f7b21fe49b33': RSS-31: Allow to skip removeEntitiesContainedInLarger in redactBetween rule and added param to sort result by positions	2022-09-09 14:07:50 +02:00
deiflaender	317a8a9af9	RSS-31: Allow to skip removeEntitiesContainedInLarger in redactBetween rule and added param to sort result by positions	2022-09-09 13:25:53 +02:00
Dominique Eiflaender	28e437a037	Pull request #464 : RSS-86: Added new rule function redactLineAfterAcrossColumns Merge in RED/redaction-service from RSS-86 to master * commit 'a5f27cfa4cf81bfe1cb29016d18d3fc40e77624d': RSS-86: Added new rule function redactLineAfterAcrossColumns	2022-09-08 13:47:45 +02:00
deiflaender	a5f27cfa4c	RSS-86: Added new rule function redactLineAfterAcrossColumns	2022-09-08 13:05:13 +02:00
Philipp Schramm	34be42cd45	Pull request #463 : RSS-2: Section: added method redactSectionText and extended redactBetween(Not) with excludeHeadline flag Merge in RED/redaction-service from RSS-2 to master * commit 'f1b5d605ccbe4984943e7452dd701004f64f7525': RSS-2: Section: added method redactSectionText and extended redactBetween(Not) with excludeHeadline flag	2022-09-08 12:17:20 +02:00
Philipp Schramm	f1b5d605cc	RSS-2: Section: added method redactSectionText and extended redactBetween(Not) with excludeHeadline flag	2022-09-08 12:10:07 +02:00
Dominique Eiflaender	c3dadd6906	Pull request #462 : RSS-42: Made redactBetween rule more generic Merge in RED/redaction-service from RSS-42 to master * commit 'c29c5eef0d05a86a18fca8978a42ffff3c42ea16': RSS-42: Made redactBetween rule more generic	2022-09-07 14:37:26 +02:00
deiflaender	c29c5eef0d	RSS-42: Made redactBetween rule more generic	2022-09-07 14:31:08 +02:00