RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -1,10 +1,12 @@
 package buildjob;
 
-import java.time.DayOfWeek;
+import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
+
 import java.time.LocalTime;
 
 import com.atlassian.bamboo.specs.api.BambooSpec;
 import com.atlassian.bamboo.specs.api.builders.BambooKey;
+import com.atlassian.bamboo.specs.api.builders.Variable;
 import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
 import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
 import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
@@ -17,20 +19,16 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
-import com.atlassian.bamboo.specs.api.builders.Variable;
+import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
 import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
 import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
 import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
-import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
 import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
 import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
-import com.atlassian.bamboo.specs.util.BambooServer;
-import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
-
+import com.atlassian.bamboo.specs.util.BambooServer;
-import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
 
 /**
  * Plan configuration for Bamboo.
@@ -57,6 +55,11 @@ public class PlanSpec {
         PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
         bambooServer.publish(planPermission);
 
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        bambooServer.publish(nightPlan);
+        PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
+        bambooServer.publish(nightPlanPermission);
+
         Plan secPlan = new PlanSpec().createSecBuild();
         bambooServer.publish(secPlan);
         PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
@@ -127,22 +130,38 @@ public class PlanSpec {
                 .linkedRepositories("RED / " + SERVICE_NAME)
 
                 .triggers(new BitbucketServerTrigger())
-                .planBranchManagement(new PlanBranchManagement()
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
-                        .createForVcsBranch()
+                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
-                        .delete(new BranchCleanup()
-                                .whenInactiveInRepositoryAfterDays(14))
                         .notificationForCommitters());
     }
 
+
+    public Plan createNightPlan() {
+
+        return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
+                .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
+                                .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
+                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                .enabled(true))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
+    }
+
+
     public Plan createSecBuild() {
-        return new Plan(
+
-                project(),
+        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
-                SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(
-                .description("Security Analysis Plan")
-                .stages(new Stage("Default Stage")
-                        .jobs(new Job("Default Job",
-                                new BambooKey("JOB1"))
-                                .tasks(
                                         new ScriptTask()
                                                 .description("Clean")
                                                 .inlineBody("#!/bin/bash\n" +
@@ -174,4 +193,5 @@ public class PlanSpec {
                         .createForVcsBranchMatching("release.*")
                         .notificationForCommitters());
     }
+
 }

bamboo-specs/src/test/java/buildjob/PlanSpecTest.java

@@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
 public class PlanSpecTest {
     @Test
     public void checkYourPlanOffline() throws PropertiesValidationException {
+
         Plan plan = new PlanSpec().createPlan();
         EntityPropertiesBuilders.build(plan);
 
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        EntityPropertiesBuilders.build(nightPlan);
+
         Plan secPlan = new PlanSpec().createSecBuild();
         EntityPropertiesBuilders.build(secPlan);
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -10,12 +10,14 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.math.BigInteger;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.MessageDigest;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -48,6 +50,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.annotation.IfProfileValue;
 import org.springframework.test.context.junit4.SpringRunner;
 
 import com.amazonaws.services.s3.AmazonS3;
@@ -86,7 +89,7 @@ import lombok.SneakyThrows;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RulesTest.RedactionIntegrationTestConfiguration.class)
+@Import(RulesTest.RulesTestConfiguration.class)
 public class RulesTest {
 
     private static final String RULES_PATH = "drools/testRules.drl";
@@ -214,6 +217,7 @@ public class RulesTest {
 
     /**
      * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
+     * If the RedactionLog already exists, the generating will be skipped
      * Test is ignored, because it's for manual tests.
      */
     @Ignore
@@ -225,18 +229,28 @@ public class RulesTest {
     }
 
 
-    /**
+    @SneakyThrows
-     * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
+    public void generateAndSaveRedactionLog(String fileName) {
-     * Test is ignored, because it's for manual tests.
-     */
-    @Ignore
-    @Test
-    public void generateRedactionLogForAllFiles() {
 
-        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
+        if (doesRedactionLogExist(fileName)) {
-        System.out.println("Will generate RedactionLog for " + files.size() + " files.");
+            System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
-        TEST_FILE_ID = "1000";
+
-        files.forEach(this::generateAndSaveRedactionLog);
+        } else {
+            generateTestFileId(fileName);
+
+            System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
+
+            loadNerForTest();
+
+            AnalyzeRequest request = prepareStorage(fileName);
+            analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+            analyzeService.analyze(request);
+
+            RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+            saveRedactionLogAsJson(redactionLog, fileName);
+
+        }
     }
 
 
@@ -254,45 +268,71 @@ public class RulesTest {
     }
 
 
+    private boolean doesRedactionLogExist(String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+        return file.exists();
+
+    }
+
+
+    @SneakyThrows
+    private void generateTestFileId(String fileName) {
+
+        MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
+        messageDigest.update(fileName.getBytes());
+        TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16);
+    }
+
+
+    /**
+     * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
+     * If a RedactionLog already exists, the generating will be skipped
+     */
+    @IfProfileValue(name = "test-groups", value = "rules-test")
+    @Test
+    public void generateRedactionLogForAllFiles() {
+
+        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
+        System.out.println("Will generate RedactionLog for " + files.size() + " files.");
+        files.forEach(this::generateAndSaveRedactionLog);
+    }
+
+
     /**
      * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
      * If RedactionLogs Json does not exist, test will fail.
-     * Test is ignored, because it's for manual tests.
      */
-    @Ignore
+    @IfProfileValue(name = "test-groups", value = "rules-test")
     @Test
     public void analyseAllFilesAndCompareRedactionLogs() {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
         System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
-        TEST_FILE_ID = "5000";
         files.forEach(this::analyseFileAndCompareRedactionLog);
     }
 
 
     @SneakyThrows
-    public void generateAndSaveRedactionLog(String fileName) {
+    private RedactionLog loadSavedRedactionLog(String pdfFileName) {
 
-        increaseTestFileId();
+        File pdfFile = new File(pdfFileName);
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+        assertThat(file).exists();
 
-        System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
+        return objectMapper.readValue(file, RedactionLog.class);
-
-        loadNerForTest();
-
-        AnalyzeRequest request = prepareStorage(fileName);
-        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
-        analyzeService.analyze(request);
-
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-
-        saveRedactionLogAsJson(redactionLog, fileName);
     }
 
 
     @SneakyThrows
     public void analyseFileAndCompareRedactionLog(String fileName) {
 
-        increaseTestFileId();
+        generateTestFileId(fileName);
         System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
 
         RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
@@ -410,25 +450,6 @@ public class RulesTest {
     }
 
 
-    private void increaseTestFileId() {
-
-        TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
-    }
-
-
-    @SneakyThrows
-    private RedactionLog loadSavedRedactionLog(String pdfFileName) {
-
-        File pdfFile = new File(pdfFileName);
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
-        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
-        File file = new File(directory, fileName);
-        assertThat(file).exists();
-
-        return objectMapper.readValue(file, RedactionLog.class);
-    }
-
-
     @SneakyThrows
     private void loadNerForTest() {
 
@@ -635,7 +656,7 @@ public class RulesTest {
 
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
-    public static class RedactionIntegrationTestConfiguration {
+    public static class RulesTestConfiguration {
 
         @Bean
         public KieContainer kieContainer() {