Merge branch 'master' into RED-4771

# Conflicts:
#	bamboo-specs/src/main/java/buildjob/PlanSpec.java

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -19,6 +19,7 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
+import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
 import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
@@ -38,7 +39,7 @@ public class PlanSpec {
 
     private static final String SERVICE_NAME = "redaction-service";
 
-    private static final String JVM_ARGS =" -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
+    private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
 
     private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
 
@@ -55,15 +56,21 @@ public class PlanSpec {
         PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
         bambooServer.publish(planPermission);
 
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        bambooServer.publish(nightPlan);
+        PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
+        bambooServer.publish(nightPlanPermission);
+
         Plan secPlan = new PlanSpec().createSecBuild();
         bambooServer.publish(secPlan);
         PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
         bambooServer.publish(secPlanPermission);
     }
 
+
     private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
-        Permissions permission = new Permissions()
+
-                .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
+        Permissions permission = new Permissions().userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
                 .groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
                 .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
                 .loggedInUserPermissions(PermissionType.VIEW)
@@ -71,105 +78,80 @@ public class PlanSpec {
         return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
     }
 
+
     private Project project() {
-        return new Project()
+
-                .name("RED")
+        return new Project().name("RED").key(new BambooKey("RED"));
-                .key(new BambooKey("RED"));
     }
 
+
     public Plan createPlan() {
-        return new Plan(
+
-                project(),
+        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
-                SERVICE_NAME, new BambooKey(SERVICE_KEY))
-                .description("Plan created from (enter repository url of your plan)")
                 .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                        .jobs(new Job("Default Job",
+                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
-                                new BambooKey("JOB1"))
+                                .cleanCheckout(true)
-                                .tasks(
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
-                                        new ScriptTask()
+                                .location(Location.FILE)
-                                                .description("Clean")
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                                .inlineBody("#!/bin/bash\n" +
+                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
-                                                        "set -e\n" +
+                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                                        "rm -rf ./*"),
+                                .enabled(true), new InjectVariablesTask().description("Inject git Tag")
-                                        new VcsCheckoutTask()
+                                .path("git.tag")
-                                                .description("Checkout Default Repository")
+                                .namespace("g")
-                                                .cleanCheckout(true)
+                                .scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                        new ScriptTask()
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                                .description("Build")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                                                .location(Location.FILE)
-                                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                                .argument(SERVICE_NAME),
-                                        createJUnitParserTask()
-                                                .description("Resultparser")
-                                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                                .enabled(true),
-                                        new InjectVariablesTask()
-                                                .description("Inject git Tag")
-                                                .path("git.tag")
-                                                .namespace("g")
-                                                .scope(InjectVariablesScope.LOCAL),
-                                        new VcsTagTask()
-                                                .description("${bamboo.g.gitTag}")
-                                                .tagName("${bamboo.g.gitTag}")
-                                                .defaultRepository())
-                                .dockerConfiguration(
-                                        new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                                .volume("/var/run/docker.sock", "/var/run/docker.sock")
-                                )
-                        )
-                )
                 .linkedRepositories("RED / " + SERVICE_NAME)
 
                 .triggers(new BitbucketServerTrigger())
-                .planBranchManagement(new PlanBranchManagement()
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
-                        .createForVcsBranch()
+                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
-                        .delete(new BranchCleanup()
-                                .whenInactiveInRepositoryAfterDays(14))
                         .notificationForCommitters());
     }
 
-    public Plan createSecBuild() {
+
-        return new Plan(
+    public Plan createNightPlan() {
-                project(),
+
-                SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
+        return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
-                .description("Security Analysis Plan")
+                .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
-                .stages(new Stage("Default Stage")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
-                        .jobs(new Job("Default Job",
+                                .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
-                                new BambooKey("JOB1"))
+                                .cleanCheckout(true)
-                                .tasks(
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
-                                        new ScriptTask()
+                                .location(Location.FILE)
-                                                .description("Clean")
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                                .inlineBody("#!/bin/bash\n" +
+                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
-                                                        "set -e\n" +
+                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                                        "rm -rf ./*"),
+                                .enabled(true))
-                                        new VcsCheckoutTask()
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .description("Checkout Default Repository")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                                .cleanCheckout(true)
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
-                                        new ScriptTask()
-                                                .description("Sonar")
-                                                .location(Location.FILE)
-                                                .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
-                                                .argument(SERVICE_NAME))
-                                .dockerConfiguration(
-                                        new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .dockerRunArguments("--net=host")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
-                                                .volume("/var/run/docker.sock", "/var/run/docker.sock")
-                                )
-                        )
-                )
                 .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
-                        new ScheduledTrigger()
+                .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
-                                .scheduleOnceDaily(LocalTime.of(23, 00)))
-                .planBranchManagement(new PlanBranchManagement()
-                        .createForVcsBranchMatching("release.*")
-                        .notificationForCommitters());
     }
+
+
+    public Plan createSecBuild() {
+
+        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
+                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Sonar")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
+                                .argument(SERVICE_NAME))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .dockerRunArguments("--net=host")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
+    }
+
 }

bamboo-specs/src/test/java/buildjob/PlanSpecTest.java

@@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
 public class PlanSpecTest {
     @Test
     public void checkYourPlanOffline() throws PropertiesValidationException {
+
         Plan plan = new PlanSpec().createPlan();
         EntityPropertiesBuilders.build(plan);
 
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        EntityPropertiesBuilders.build(nightPlan);
+
         Plan secPlan = new PlanSpec().createSecBuild();
         EntityPropertiesBuilders.build(secPlan);
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -10,12 +10,14 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.math.BigInteger;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.MessageDigest;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -48,6 +50,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.annotation.IfProfileValue;
 import org.springframework.test.context.junit4.SpringRunner;
 
 import com.amazonaws.services.s3.AmazonS3;
@@ -86,7 +89,7 @@ import lombok.SneakyThrows;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RulesTest.RedactionIntegrationTestConfiguration.class)
+@Import(RulesTest.RulesTestConfiguration.class)
 public class RulesTest {
 
     private static final String RULES_PATH = "drools/testRules.drl";
@@ -151,34 +154,6 @@ public class RulesTest {
     private LegalBasisClient legalBasisClient;
 
 
-    @After
-    public void cleanupStorage() {
-
-        if (this.storageService instanceof FileSystemBackedStorageService) {
-            ((FileSystemBackedStorageService) this.storageService).clearStorage();
-        }
-    }
-
-
-    private static String loadFromClassPath(String path) {
-
-        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
-        }
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
-            StringBuilder sb = new StringBuilder();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
-        }
-    }
-
-
     @Before
     public void stubClients() {
 
@@ -201,9 +176,7 @@ public class RulesTest {
                 .hexColor("#ffe187")
                 .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
                 .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
-                .build()));
 
         mockDictionaryCalls(null);
         mockDictionaryCalls(0L);
@@ -212,8 +185,18 @@ public class RulesTest {
     }
 
 
+    @After
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
     /**
      * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
+     * If the RedactionLog already exists, the generating will be skipped
      * Test is ignored, because it's for manual tests.
      */
     @Ignore
@@ -227,15 +210,14 @@ public class RulesTest {
 
     /**
      * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
-     * Test is ignored, because it's for manual tests.
+     * If a RedactionLog already exists, the generating will be skipped
      */
-    @Ignore
+    @IfProfileValue(name = "test-groups", value = "rules-test")
     @Test
     public void generateRedactionLogForAllFiles() {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
         System.out.println("Will generate RedactionLog for " + files.size() + " files.");
-        TEST_FILE_ID = "1000";
         files.forEach(this::generateAndSaveRedactionLog);
     }
 
@@ -257,42 +239,46 @@ public class RulesTest {
     /**
      * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
      * If RedactionLogs Json does not exist, test will fail.
-     * Test is ignored, because it's for manual tests.
      */
-    @Ignore
+    @IfProfileValue(name = "test-groups", value = "rules-test")
     @Test
     public void analyseAllFilesAndCompareRedactionLogs() {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
         System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
-        TEST_FILE_ID = "5000";
         files.forEach(this::analyseFileAndCompareRedactionLog);
     }
 
 
     @SneakyThrows
-    public void generateAndSaveRedactionLog(String fileName) {
+    private void generateAndSaveRedactionLog(String fileName) {
 
-        increaseTestFileId();
+        if (doesRedactionLogExist(fileName)) {
+            System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
 
-        System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
+        } else {
+            generateTestFileId(fileName);
 
-        loadNerForTest();
+            System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
 
-        AnalyzeRequest request = prepareStorage(fileName);
+            loadNerForTest();
-        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
-        analyzeService.analyze(request);
 
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+            AnalyzeRequest request = prepareStorage(fileName);
+            analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+            analyzeService.analyze(request);
 
-        saveRedactionLogAsJson(redactionLog, fileName);
+            RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+            saveRedactionLogAsJson(redactionLog, fileName);
+
+        }
     }
 
 
     @SneakyThrows
     public void analyseFileAndCompareRedactionLog(String fileName) {
 
-        increaseTestFileId();
+        generateTestFileId(fileName);
         System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
 
         RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
@@ -410,9 +396,44 @@ public class RulesTest {
     }
 
 
-    private void increaseTestFileId() {
+    private boolean doesRedactionLogExist(String pdfFileName) {
 
-        TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
+        File pdfFile = new File(pdfFileName);
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+        return file.exists();
+
+    }
+
+
+    @SneakyThrows
+    private void generateTestFileId(String fileName) {
+
+        MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
+        messageDigest.update(fileName.getBytes());
+        TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16);
+    }
+
+
+    @SneakyThrows
+    private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        File dr = new File(directory);
+        boolean created = dr.mkdirs();
+        if (created) {
+            System.out.println("Directory was created");
+        }
+
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+
+        objectMapper.writeValue(file, redactionLog);
+
+        System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
     }
 
 
@@ -466,27 +487,6 @@ public class RulesTest {
     }
 
 
-    @SneakyThrows
-    private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
-
-        File pdfFile = new File(pdfFileName);
-
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
-        File dr = new File(directory);
-        boolean created = dr.mkdirs();
-        if (created) {
-            System.out.println("Directory was created");
-        }
-
-        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
-        File file = new File(directory, fileName);
-
-        objectMapper.writeValue(file, redactionLog);
-
-        System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
-    }
-
-
     private void loadDictionaryForTest() {
 
         dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
@@ -633,9 +633,28 @@ public class RulesTest {
     }
 
 
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
-    public static class RedactionIntegrationTestConfiguration {
+    public static class RulesTestConfiguration {
 
         @Bean
         public KieContainer kieContainer() {