Pull request #430: RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist

Merge in RED/redaction-service from RED-4510 to master

* commit 'db8eedc9a36dfba22946940b31e28854ffdca26d':
  RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -1,10 +1,12 @@
 package buildjob;
 
-import java.time.DayOfWeek;
+import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
+
 import java.time.LocalTime;
 
 import com.atlassian.bamboo.specs.api.BambooSpec;
 import com.atlassian.bamboo.specs.api.builders.BambooKey;
+import com.atlassian.bamboo.specs.api.builders.Variable;
 import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
 import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
 import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
@@ -17,20 +19,16 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
-import com.atlassian.bamboo.specs.api.builders.Variable;
+import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
 import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
 import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
 import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
-import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
 import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
 import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
-import com.atlassian.bamboo.specs.util.BambooServer;
-import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
-
+import com.atlassian.bamboo.specs.util.BambooServer;
-import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
 
 /**
  * Plan configuration for Bamboo.
@@ -57,6 +55,11 @@ public class PlanSpec {
         PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
         bambooServer.publish(planPermission);
 
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        bambooServer.publish(nightPlan);
+        PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
+        bambooServer.publish(nightPlanPermission);
+
         Plan secPlan = new PlanSpec().createSecBuild();
         bambooServer.publish(secPlan);
         PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
@@ -127,22 +130,38 @@ public class PlanSpec {
                 .linkedRepositories("RED / " + SERVICE_NAME)
 
                 .triggers(new BitbucketServerTrigger())
-                .planBranchManagement(new PlanBranchManagement()
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
-                        .createForVcsBranch()
+                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
-                        .delete(new BranchCleanup()
-                                .whenInactiveInRepositoryAfterDays(14))
                         .notificationForCommitters());
     }
 
+
+    public Plan createNightPlan() {
+
+        return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
+                .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
+                                .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
+                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                .enabled(true))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
+    }
+
+
     public Plan createSecBuild() {
-        return new Plan(
+
-                project(),
+        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
-                SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(
-                .description("Security Analysis Plan")
-                .stages(new Stage("Default Stage")
-                        .jobs(new Job("Default Job",
-                                new BambooKey("JOB1"))
-                                .tasks(
                                         new ScriptTask()
                                                 .description("Clean")
                                                 .inlineBody("#!/bin/bash\n" +
@@ -174,4 +193,5 @@ public class PlanSpec {
                         .createForVcsBranchMatching("release.*")
                         .notificationForCommitters());
     }
+
 }

bamboo-specs/src/test/java/buildjob/PlanSpecTest.java

@@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
 public class PlanSpecTest {
     @Test
     public void checkYourPlanOffline() throws PropertiesValidationException {
+
         Plan plan = new PlanSpec().createPlan();
         EntityPropertiesBuilders.build(plan);
 
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        EntityPropertiesBuilders.build(nightPlan);
+
         Plan secPlan = new PlanSpec().createSecBuild();
         EntityPropertiesBuilders.build(secPlan);
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -10,12 +10,14 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.math.BigInteger;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.MessageDigest;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -48,6 +50,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.annotation.IfProfileValue;
 import org.springframework.test.context.junit4.SpringRunner;
 
 import com.amazonaws.services.s3.AmazonS3;
@@ -86,7 +89,7 @@ import lombok.SneakyThrows;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RulesTest.RedactionIntegrationTestConfiguration.class)
+@Import(RulesTest.RulesTestConfiguration.class)
 public class RulesTest {
 
     private static final String RULES_PATH = "drools/testRules.drl";
@@ -151,34 +154,6 @@ public class RulesTest {
     private LegalBasisClient legalBasisClient;
 
 
-    @After
-    public void cleanupStorage() {
-
-        if (this.storageService instanceof FileSystemBackedStorageService) {
-            ((FileSystemBackedStorageService) this.storageService).clearStorage();
-        }
-    }
-
-
-    private static String loadFromClassPath(String path) {
-
-        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
-        }
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
-            StringBuilder sb = new StringBuilder();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
-        }
-    }
-
-
     @Before
     public void stubClients() {
 
@@ -212,8 +187,18 @@ public class RulesTest {
     }
 
 
+    @After
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
     /**
      * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
+     * If the RedactionLog already exists, the generating will be skipped
      * Test is ignored, because it's for manual tests.
      */
     @Ignore
@@ -227,15 +212,14 @@ public class RulesTest {
 
     /**
      * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
-     * Test is ignored, because it's for manual tests.
+     * If a RedactionLog already exists, the generating will be skipped
      */
-    @Ignore
+    @IfProfileValue(name = "test-groups", value = "rules-test")
     @Test
     public void generateRedactionLogForAllFiles() {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
         System.out.println("Will generate RedactionLog for " + files.size() + " files.");
-        TEST_FILE_ID = "1000";
         files.forEach(this::generateAndSaveRedactionLog);
     }
 
@@ -257,23 +241,25 @@ public class RulesTest {
     /**
      * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
      * If RedactionLogs Json does not exist, test will fail.
-     * Test is ignored, because it's for manual tests.
      */
-    @Ignore
+    @IfProfileValue(name = "test-groups", value = "rules-test")
     @Test
     public void analyseAllFilesAndCompareRedactionLogs() {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
         System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
-        TEST_FILE_ID = "5000";
         files.forEach(this::analyseFileAndCompareRedactionLog);
     }
 
 
     @SneakyThrows
-    public void generateAndSaveRedactionLog(String fileName) {
+    private void generateAndSaveRedactionLog(String fileName) {
 
-        increaseTestFileId();
+        if (doesRedactionLogExist(fileName)) {
+            System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
+
+        } else {
+            generateTestFileId(fileName);
 
             System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
 
@@ -286,13 +272,15 @@ public class RulesTest {
             RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
             saveRedactionLogAsJson(redactionLog, fileName);
+
+        }
     }
 
 
     @SneakyThrows
     public void analyseFileAndCompareRedactionLog(String fileName) {
 
-        increaseTestFileId();
+        generateTestFileId(fileName);
         System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
 
         RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
@@ -410,9 +398,44 @@ public class RulesTest {
     }
 
 
-    private void increaseTestFileId() {
+    private boolean doesRedactionLogExist(String pdfFileName) {
 
-        TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
+        File pdfFile = new File(pdfFileName);
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+        return file.exists();
+
+    }
+
+
+    @SneakyThrows
+    private void generateTestFileId(String fileName) {
+
+        MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
+        messageDigest.update(fileName.getBytes());
+        TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16);
+    }
+
+
+    @SneakyThrows
+    private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        File dr = new File(directory);
+        boolean created = dr.mkdirs();
+        if (created) {
+            System.out.println("Directory was created");
+        }
+
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+
+        objectMapper.writeValue(file, redactionLog);
+
+        System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
     }
 
 
@@ -466,27 +489,6 @@ public class RulesTest {
     }
 
 
-    @SneakyThrows
-    private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
-
-        File pdfFile = new File(pdfFileName);
-
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
-        File dr = new File(directory);
-        boolean created = dr.mkdirs();
-        if (created) {
-            System.out.println("Directory was created");
-        }
-
-        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
-        File file = new File(directory, fileName);
-
-        objectMapper.writeValue(file, redactionLog);
-
-        System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
-    }
-
-
     private void loadDictionaryForTest() {
 
         dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
@@ -633,9 +635,28 @@ public class RulesTest {
     }
 
 
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
-    public static class RedactionIntegrationTestConfiguration {
+    public static class RulesTestConfiguration {
 
         @Bean
         public KieContainer kieContainer() {