diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java index 451c1f13..0090882a 100644 --- a/bamboo-specs/src/main/java/buildjob/PlanSpec.java +++ b/bamboo-specs/src/main/java/buildjob/PlanSpec.java @@ -19,6 +19,7 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup; import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement; import com.atlassian.bamboo.specs.api.builders.project.Project; import com.atlassian.bamboo.specs.builders.task.CheckoutItem; +import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask; import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask; import com.atlassian.bamboo.specs.builders.task.ScriptTask; import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask; @@ -38,7 +39,7 @@ public class PlanSpec { private static final String SERVICE_NAME = "redaction-service"; - private static final String JVM_ARGS =" -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 "; + private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 "; private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", ""); @@ -55,15 +56,21 @@ public class PlanSpec { PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier()); bambooServer.publish(planPermission); + Plan nightPlan = new PlanSpec().createNightPlan(); + bambooServer.publish(nightPlan); + PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier()); + bambooServer.publish(nightPlanPermission); + Plan secPlan = new PlanSpec().createSecBuild(); bambooServer.publish(secPlan); PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier()); bambooServer.publish(secPlanPermission); } + private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) { - Permissions permission = new Permissions() - .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD) + + Permissions permission = new Permissions().userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD) .groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) .loggedInUserPermissions(PermissionType.VIEW) @@ -71,105 +78,80 @@ public class PlanSpec { return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission); } + private Project project() { - return new Project() - .name("RED") - .key(new BambooKey("RED")); + + return new Project().name("RED").key(new BambooKey("RED")); } + public Plan createPlan() { - return new Plan( - project(), - SERVICE_NAME, new BambooKey(SERVICE_KEY)) - .description("Plan created from (enter repository url of your plan)") + + return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)") .variables(new Variable("maven_add_param", "")) - .stages(new Stage("Default Stage") - .jobs(new Job("Default Job", - new BambooKey("JOB1")) - .tasks( - new ScriptTask() - .description("Clean") - .inlineBody("#!/bin/bash\n" + - "set -e\n" + - "rm -rf ./*"), - new VcsCheckoutTask() - .description("Checkout Default Repository") - .cleanCheckout(true) - .checkoutItems(new CheckoutItem().defaultRepository()), - new ScriptTask() - .description("Build") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh") - .argument(SERVICE_NAME), - createJUnitParserTask() - .description("Resultparser") - .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml") - .enabled(true), - new InjectVariablesTask() - .description("Inject git Tag") - .path("git.tag") - .namespace("g") - .scope(InjectVariablesScope.LOCAL), - new VcsTagTask() - .description("${bamboo.g.gitTag}") - .tagName("${bamboo.g.gitTag}") - .defaultRepository()) - .dockerConfiguration( - new DockerConfiguration() - .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") - .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") - .volume("/var/run/docker.sock", "/var/run/docker.sock") - ) - ) - ) + .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean") + .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository") + .cleanCheckout(true) + .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh") + .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser") + .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml") + .enabled(true), new InjectVariablesTask().description("Inject git Tag") + .path("git.tag") + .namespace("g") + .scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository()) + .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") + .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) .linkedRepositories("RED / " + SERVICE_NAME) .triggers(new BitbucketServerTrigger()) - .planBranchManagement(new PlanBranchManagement() - .createForVcsBranch() - .delete(new BranchCleanup() - .whenInactiveInRepositoryAfterDays(14)) + .planBranchManagement(new PlanBranchManagement().createForVcsBranch() + .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)) .notificationForCommitters()); } - public Plan createSecBuild() { - return new Plan( - project(), - SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")) - .description("Security Analysis Plan") - .stages(new Stage("Default Stage") - .jobs(new Job("Default Job", - new BambooKey("JOB1")) - .tasks( - new ScriptTask() - .description("Clean") - .inlineBody("#!/bin/bash\n" + - "set -e\n" + - "rm -rf ./*"), - new VcsCheckoutTask() - .description("Checkout Default Repository") - .cleanCheckout(true) - .checkoutItems(new CheckoutItem().defaultRepository()), - new ScriptTask() - .description("Sonar") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh") - .argument(SERVICE_NAME)) - .dockerConfiguration( - new DockerConfiguration() - .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") - .dockerRunArguments("--net=host") - .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml") - .volume("/var/run/docker.sock", "/var/run/docker.sock") - ) - ) - ) + + public Plan createNightPlan() { + + return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests") + .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test")) + .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.") + .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository") + .cleanCheckout(true) + .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh") + .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser") + .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml") + .enabled(true)) + .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") + .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) .linkedRepositories("RED / " + SERVICE_NAME) - .triggers( - new ScheduledTrigger() - .scheduleOnceDaily(LocalTime.of(23, 00))) - .planBranchManagement(new PlanBranchManagement() - .createForVcsBranchMatching("release.*") - .notificationForCommitters()); + .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00))) + .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters()); } + + + public Plan createSecBuild() { + + return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan") + .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean") + .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository") + .cleanCheckout(true) + .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Sonar") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh") + .argument(SERVICE_NAME)) + .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") + .dockerRunArguments("--net=host") + .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) + .linkedRepositories("RED / " + SERVICE_NAME) + .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00))) + .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters()); + } + } diff --git a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java index ed095c7c..bb483899 100644 --- a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java +++ b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java @@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders; public class PlanSpecTest { @Test public void checkYourPlanOffline() throws PropertiesValidationException { + Plan plan = new PlanSpec().createPlan(); EntityPropertiesBuilders.build(plan); + Plan nightPlan = new PlanSpec().createNightPlan(); + EntityPropertiesBuilders.build(nightPlan); + Plan secPlan = new PlanSpec().createSecBuild(); EntityPropertiesBuilders.build(secPlan); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index b4966821..8741174e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -10,12 +10,14 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.math.BigInteger; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashMap; @@ -48,6 +50,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.core.io.ClassPathResource; +import org.springframework.test.annotation.IfProfileValue; import org.springframework.test.context.junit4.SpringRunner; import com.amazonaws.services.s3.AmazonS3; @@ -86,7 +89,7 @@ import lombok.SneakyThrows; @RunWith(SpringRunner.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(RulesTest.RedactionIntegrationTestConfiguration.class) +@Import(RulesTest.RulesTestConfiguration.class) public class RulesTest { private static final String RULES_PATH = "drools/testRules.drl"; @@ -151,34 +154,6 @@ public class RulesTest { private LegalBasisClient legalBasisClient; - @After - public void cleanupStorage() { - - if (this.storageService instanceof FileSystemBackedStorageService) { - ((FileSystemBackedStorageService) this.storageService).clearStorage(); - } - } - - - private static String loadFromClassPath(String path) { - - URL resource = ResourceLoader.class.getClassLoader().getResource(path); - if (resource == null) { - throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); - } - try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { - StringBuilder sb = new StringBuilder(); - String str; - while ((str = br.readLine()) != null) { - sb.append(str).append("\n"); - } - return sb.toString(); - } catch (IOException e) { - throw new IllegalArgumentException("could not load classpath resource: " + path, e); - } - } - - @Before public void stubClients() { @@ -201,9 +176,7 @@ public class RulesTest { .hexColor("#ffe187") .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) - .build())); + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build())); mockDictionaryCalls(null); mockDictionaryCalls(0L); @@ -212,8 +185,18 @@ public class RulesTest { } + @After + public void cleanupStorage() { + + if (this.storageService instanceof FileSystemBackedStorageService) { + ((FileSystemBackedStorageService) this.storageService).clearStorage(); + } + } + + /** * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH. + * If the RedactionLog already exists, the generating will be skipped * Test is ignored, because it's for manual tests. */ @Ignore @@ -227,15 +210,14 @@ public class RulesTest { /** * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH. - * Test is ignored, because it's for manual tests. + * If a RedactionLog already exists, the generating will be skipped */ - @Ignore + @IfProfileValue(name = "test-groups", value = "rules-test") @Test public void generateRedactionLogForAllFiles() { Set files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH)); System.out.println("Will generate RedactionLog for " + files.size() + " files."); - TEST_FILE_ID = "1000"; files.forEach(this::generateAndSaveRedactionLog); } @@ -257,42 +239,46 @@ public class RulesTest { /** * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH. * If RedactionLogs Json does not exist, test will fail. - * Test is ignored, because it's for manual tests. */ - @Ignore + @IfProfileValue(name = "test-groups", value = "rules-test") @Test public void analyseAllFilesAndCompareRedactionLogs() { Set files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH)); System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs."); - TEST_FILE_ID = "5000"; files.forEach(this::analyseFileAndCompareRedactionLog); } @SneakyThrows - public void generateAndSaveRedactionLog(String fileName) { + private void generateAndSaveRedactionLog(String fileName) { - increaseTestFileId(); + if (doesRedactionLogExist(fileName)) { + System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists."); - System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID); + } else { + generateTestFileId(fileName); - loadNerForTest(); + System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID); - AnalyzeRequest request = prepareStorage(fileName); - analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); - analyzeService.analyze(request); + loadNerForTest(); - RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + AnalyzeRequest request = prepareStorage(fileName); + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + analyzeService.analyze(request); - saveRedactionLogAsJson(redactionLog, fileName); + RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + saveRedactionLogAsJson(redactionLog, fileName); + + } } @SneakyThrows public void analyseFileAndCompareRedactionLog(String fileName) { - increaseTestFileId(); + generateTestFileId(fileName); System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog"); RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName); @@ -410,9 +396,44 @@ public class RulesTest { } - private void increaseTestFileId() { + private boolean doesRedactionLogExist(String pdfFileName) { - TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1); + File pdfFile = new File(pdfFileName); + String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath(); + String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json"); + File file = new File(directory, fileName); + return file.exists(); + + } + + + @SneakyThrows + private void generateTestFileId(String fileName) { + + MessageDigest messageDigest = MessageDigest.getInstance("SHA-256"); + messageDigest.update(fileName.getBytes()); + TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16); + } + + + @SneakyThrows + private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) { + + File pdfFile = new File(pdfFileName); + + String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath(); + File dr = new File(directory); + boolean created = dr.mkdirs(); + if (created) { + System.out.println("Directory was created"); + } + + String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json"); + File file = new File(directory, fileName); + + objectMapper.writeValue(file, redactionLog); + + System.out.println("Saved RedactionLog for " + fileName + " here " + directory); } @@ -466,27 +487,6 @@ public class RulesTest { } - @SneakyThrows - private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) { - - File pdfFile = new File(pdfFileName); - - String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath(); - File dr = new File(directory); - boolean created = dr.mkdirs(); - if (created) { - System.out.println("Directory was created"); - } - - String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json"); - File file = new File(directory, fileName); - - objectMapper.writeValue(file, redactionLog); - - System.out.println("Saved RedactionLog for " + fileName + " here " + directory); - } - - private void loadDictionaryForTest() { dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) @@ -633,9 +633,28 @@ public class RulesTest { } + private static String loadFromClassPath(String path) { + + URL resource = ResourceLoader.class.getClassLoader().getResource(path); + if (resource == null) { + throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); + } + try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String str; + while ((str = br.readLine()) != null) { + sb.append(str).append("\n"); + } + return sb.toString(); + } catch (IOException e) { + throw new IllegalArgumentException("could not load classpath resource: " + path, e); + } + } + + @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class}) - public static class RedactionIntegrationTestConfiguration { + public static class RulesTestConfiguration { @Bean public KieContainer kieContainer() {