From db8eedc9a36dfba22946940b31e28854ffdca26d Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Mon, 25 Jul 2022 14:08:35 +0200 Subject: [PATCH] RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist --- .../src/main/java/buildjob/PlanSpec.java | 58 ++++--- .../src/test/java/buildjob/PlanSpecTest.java | 4 + .../redaction/v1/server/RulesTest.java | 159 ++++++++++-------- 3 files changed, 133 insertions(+), 88 deletions(-) diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java index f90db7ac..d391f489 100644 --- a/bamboo-specs/src/main/java/buildjob/PlanSpec.java +++ b/bamboo-specs/src/main/java/buildjob/PlanSpec.java @@ -1,10 +1,12 @@ package buildjob; -import java.time.DayOfWeek; +import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask; + import java.time.LocalTime; import com.atlassian.bamboo.specs.api.BambooSpec; import com.atlassian.bamboo.specs.api.builders.BambooKey; +import com.atlassian.bamboo.specs.api.builders.Variable; import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration; import com.atlassian.bamboo.specs.api.builders.permission.PermissionType; import com.atlassian.bamboo.specs.api.builders.permission.Permissions; @@ -17,20 +19,16 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup; import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement; import com.atlassian.bamboo.specs.api.builders.project.Project; import com.atlassian.bamboo.specs.builders.task.CheckoutItem; -import com.atlassian.bamboo.specs.api.builders.Variable; +import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask; import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask; import com.atlassian.bamboo.specs.builders.task.ScriptTask; import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask; import com.atlassian.bamboo.specs.builders.task.VcsTagTask; import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger; -import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger; import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger; import com.atlassian.bamboo.specs.model.task.InjectVariablesScope; -import com.atlassian.bamboo.specs.util.BambooServer; -import com.atlassian.bamboo.specs.builders.task.ScriptTask; import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location; - -import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask; +import com.atlassian.bamboo.specs.util.BambooServer; /** * Plan configuration for Bamboo. @@ -57,6 +55,11 @@ public class PlanSpec { PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier()); bambooServer.publish(planPermission); + Plan nightPlan = new PlanSpec().createNightPlan(); + bambooServer.publish(nightPlan); + PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier()); + bambooServer.publish(nightPlanPermission); + Plan secPlan = new PlanSpec().createSecBuild(); bambooServer.publish(secPlan); PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier()); @@ -127,22 +130,38 @@ public class PlanSpec { .linkedRepositories("RED / " + SERVICE_NAME) .triggers(new BitbucketServerTrigger()) - .planBranchManagement(new PlanBranchManagement() - .createForVcsBranch() - .delete(new BranchCleanup() - .whenInactiveInRepositoryAfterDays(14)) + .planBranchManagement(new PlanBranchManagement().createForVcsBranch() + .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)) .notificationForCommitters()); } + + public Plan createNightPlan() { + + return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests") + .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test")) + .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.") + .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository") + .cleanCheckout(true) + .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh") + .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser") + .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml") + .enabled(true)) + .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") + .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) + .linkedRepositories("RED / " + SERVICE_NAME) + .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00))) + .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters()); + } + + public Plan createSecBuild() { - return new Plan( - project(), - SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")) - .description("Security Analysis Plan") - .stages(new Stage("Default Stage") - .jobs(new Job("Default Job", - new BambooKey("JOB1")) - .tasks( + + return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan") + .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks( new ScriptTask() .description("Clean") .inlineBody("#!/bin/bash\n" + @@ -174,4 +193,5 @@ public class PlanSpec { .createForVcsBranchMatching("release.*") .notificationForCommitters()); } + } diff --git a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java index ed095c7c..bb483899 100644 --- a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java +++ b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java @@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders; public class PlanSpecTest { @Test public void checkYourPlanOffline() throws PropertiesValidationException { + Plan plan = new PlanSpec().createPlan(); EntityPropertiesBuilders.build(plan); + Plan nightPlan = new PlanSpec().createNightPlan(); + EntityPropertiesBuilders.build(nightPlan); + Plan secPlan = new PlanSpec().createSecBuild(); EntityPropertiesBuilders.build(secPlan); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index b4966821..94c7e362 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -10,12 +10,14 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.math.BigInteger; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashMap; @@ -48,6 +50,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.core.io.ClassPathResource; +import org.springframework.test.annotation.IfProfileValue; import org.springframework.test.context.junit4.SpringRunner; import com.amazonaws.services.s3.AmazonS3; @@ -86,7 +89,7 @@ import lombok.SneakyThrows; @RunWith(SpringRunner.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(RulesTest.RedactionIntegrationTestConfiguration.class) +@Import(RulesTest.RulesTestConfiguration.class) public class RulesTest { private static final String RULES_PATH = "drools/testRules.drl"; @@ -151,34 +154,6 @@ public class RulesTest { private LegalBasisClient legalBasisClient; - @After - public void cleanupStorage() { - - if (this.storageService instanceof FileSystemBackedStorageService) { - ((FileSystemBackedStorageService) this.storageService).clearStorage(); - } - } - - - private static String loadFromClassPath(String path) { - - URL resource = ResourceLoader.class.getClassLoader().getResource(path); - if (resource == null) { - throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); - } - try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { - StringBuilder sb = new StringBuilder(); - String str; - while ((str = br.readLine()) != null) { - sb.append(str).append("\n"); - } - return sb.toString(); - } catch (IOException e) { - throw new IllegalArgumentException("could not load classpath resource: " + path, e); - } - } - - @Before public void stubClients() { @@ -212,8 +187,18 @@ public class RulesTest { } + @After + public void cleanupStorage() { + + if (this.storageService instanceof FileSystemBackedStorageService) { + ((FileSystemBackedStorageService) this.storageService).clearStorage(); + } + } + + /** * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH. + * If the RedactionLog already exists, the generating will be skipped * Test is ignored, because it's for manual tests. */ @Ignore @@ -227,15 +212,14 @@ public class RulesTest { /** * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH. - * Test is ignored, because it's for manual tests. + * If a RedactionLog already exists, the generating will be skipped */ - @Ignore + @IfProfileValue(name = "test-groups", value = "rules-test") @Test public void generateRedactionLogForAllFiles() { Set files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH)); System.out.println("Will generate RedactionLog for " + files.size() + " files."); - TEST_FILE_ID = "1000"; files.forEach(this::generateAndSaveRedactionLog); } @@ -257,42 +241,46 @@ public class RulesTest { /** * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH. * If RedactionLogs Json does not exist, test will fail. - * Test is ignored, because it's for manual tests. */ - @Ignore + @IfProfileValue(name = "test-groups", value = "rules-test") @Test public void analyseAllFilesAndCompareRedactionLogs() { Set files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH)); System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs."); - TEST_FILE_ID = "5000"; files.forEach(this::analyseFileAndCompareRedactionLog); } @SneakyThrows - public void generateAndSaveRedactionLog(String fileName) { + private void generateAndSaveRedactionLog(String fileName) { - increaseTestFileId(); + if (doesRedactionLogExist(fileName)) { + System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists."); - System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID); + } else { + generateTestFileId(fileName); - loadNerForTest(); + System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID); - AnalyzeRequest request = prepareStorage(fileName); - analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); - analyzeService.analyze(request); + loadNerForTest(); - RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + AnalyzeRequest request = prepareStorage(fileName); + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + analyzeService.analyze(request); - saveRedactionLogAsJson(redactionLog, fileName); + RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + saveRedactionLogAsJson(redactionLog, fileName); + + } } @SneakyThrows public void analyseFileAndCompareRedactionLog(String fileName) { - increaseTestFileId(); + generateTestFileId(fileName); System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog"); RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName); @@ -410,9 +398,44 @@ public class RulesTest { } - private void increaseTestFileId() { + private boolean doesRedactionLogExist(String pdfFileName) { - TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1); + File pdfFile = new File(pdfFileName); + String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath(); + String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json"); + File file = new File(directory, fileName); + return file.exists(); + + } + + + @SneakyThrows + private void generateTestFileId(String fileName) { + + MessageDigest messageDigest = MessageDigest.getInstance("SHA-256"); + messageDigest.update(fileName.getBytes()); + TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16); + } + + + @SneakyThrows + private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) { + + File pdfFile = new File(pdfFileName); + + String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath(); + File dr = new File(directory); + boolean created = dr.mkdirs(); + if (created) { + System.out.println("Directory was created"); + } + + String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json"); + File file = new File(directory, fileName); + + objectMapper.writeValue(file, redactionLog); + + System.out.println("Saved RedactionLog for " + fileName + " here " + directory); } @@ -466,27 +489,6 @@ public class RulesTest { } - @SneakyThrows - private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) { - - File pdfFile = new File(pdfFileName); - - String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath(); - File dr = new File(directory); - boolean created = dr.mkdirs(); - if (created) { - System.out.println("Directory was created"); - } - - String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json"); - File file = new File(directory, fileName); - - objectMapper.writeValue(file, redactionLog); - - System.out.println("Saved RedactionLog for " + fileName + " here " + directory); - } - - private void loadDictionaryForTest() { dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) @@ -633,9 +635,28 @@ public class RulesTest { } + private static String loadFromClassPath(String path) { + + URL resource = ResourceLoader.class.getClassLoader().getResource(path); + if (resource == null) { + throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); + } + try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String str; + while ((str = br.readLine()) != null) { + sb.append(str).append("\n"); + } + return sb.toString(); + } catch (IOException e) { + throw new IllegalArgumentException("could not load classpath resource: " + path, e); + } + } + + @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class}) - public static class RedactionIntegrationTestConfiguration { + public static class RulesTestConfiguration { @Bean public KieContainer kieContainer() {