Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist

Browse Source
This commit is contained in:
Philipp Schramm 2022-07-25 14:08:35 +02:00
parent 58e16fe39a
commit db8eedc9a3
3 changed files with 133 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
bamboo-specs/src/main/java/buildjob/PlanSpec.java
View File

@ -1,10 +1,12 @@
package buildjob;
import java.time.DayOfWeek;
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
import java.time.LocalTime;
import com.atlassian.bamboo.specs.api.BambooSpec;
import com.atlassian.bamboo.specs.api.builders.BambooKey;
import com.atlassian.bamboo.specs.api.builders.Variable;
import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
@ -17,20 +19,16 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
import com.atlassian.bamboo.specs.api.builders.project.Project;
import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
import com.atlassian.bamboo.specs.api.builders.Variable;
import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
import com.atlassian.bamboo.specs.util.BambooServer;
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
import com.atlassian.bamboo.specs.util.BambooServer;
/**
* Plan configuration for Bamboo.
@ -57,6 +55,11 @@ public class PlanSpec {
PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
bambooServer.publish(planPermission);
Plan nightPlan = new PlanSpec().createNightPlan();
bambooServer.publish(nightPlan);
PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
bambooServer.publish(nightPlanPermission);
Plan secPlan = new PlanSpec().createSecBuild();
bambooServer.publish(secPlan);
PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
@ -127,22 +130,38 @@ public class PlanSpec {
.linkedRepositories("RED / " + SERVICE_NAME)
.triggers(new BitbucketServerTrigger())
.planBranchManagement(new PlanBranchManagement()
.createForVcsBranch()
.delete(new BranchCleanup()
.whenInactiveInRepositoryAfterDays(14))
.planBranchManagement(new PlanBranchManagement().createForVcsBranch()
.delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
.notificationForCommitters());
}
public Plan createNightPlan() {
return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
.variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
.enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
.cleanCheckout(true)
.checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
.location(Location.FILE)
.fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
.argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
.enabled(true))
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
.linkedRepositories("RED / " + SERVICE_NAME)
.triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
.planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
}
public Plan createSecBuild() {
return new Plan(
project(),
SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
.description("Security Analysis Plan")
.stages(new Stage("Default Stage")
.jobs(new Job("Default Job",
new BambooKey("JOB1"))
.tasks(
return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(
new ScriptTask()
.description("Clean")
.inlineBody("#!/bin/bash\n" +
@ -174,4 +193,5 @@ public class PlanSpec {
.createForVcsBranchMatching("release.*")
.notificationForCommitters());
}
}

4
bamboo-specs/src/test/java/buildjob/PlanSpecTest.java
View File

@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
public class PlanSpecTest {
@Test
public void checkYourPlanOffline() throws PropertiesValidationException {
Plan plan = new PlanSpec().createPlan();
EntityPropertiesBuilders.build(plan);
Plan nightPlan = new PlanSpec().createNightPlan();
EntityPropertiesBuilders.build(nightPlan);
Plan secPlan = new PlanSpec().createSecBuild();
EntityPropertiesBuilders.build(secPlan);
}

159
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -10,12 +10,14 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigInteger;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
@ -48,6 +50,7 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.annotation.IfProfileValue;
import org.springframework.test.context.junit4.SpringRunner;
import com.amazonaws.services.s3.AmazonS3;
@ -86,7 +89,7 @@ import lombok.SneakyThrows;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RulesTest.RedactionIntegrationTestConfiguration.class)
@Import(RulesTest.RulesTestConfiguration.class)
public class RulesTest {
private static final String RULES_PATH = "drools/testRules.drl";
@ -151,34 +154,6 @@ public class RulesTest {
private LegalBasisClient legalBasisClient;
@After
public void cleanupStorage() {
if (this.storageService instanceof FileSystemBackedStorageService) {
((FileSystemBackedStorageService) this.storageService).clearStorage();
}
}
private static String loadFromClassPath(String path) {
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
if (resource == null) {
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
}
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
StringBuilder sb = new StringBuilder();
String str;
while ((str = br.readLine()) != null) {
sb.append(str).append("\n");
}
return sb.toString();
} catch (IOException e) {
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
}
}
@Before
public void stubClients() {
@ -212,8 +187,18 @@ public class RulesTest {
}
@After
public void cleanupStorage() {
if (this.storageService instanceof FileSystemBackedStorageService) {
((FileSystemBackedStorageService) this.storageService).clearStorage();
}
}
/**
* Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
* If the RedactionLog already exists, the generating will be skipped
* Test is ignored, because it's for manual tests.
*/
@Ignore
@ -227,15 +212,14 @@ public class RulesTest {
/**
* Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
* Test is ignored, because it's for manual tests.
* If a RedactionLog already exists, the generating will be skipped
*/
@Ignore
@IfProfileValue(name = "test-groups", value = "rules-test")
@Test
public void generateRedactionLogForAllFiles() {
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
System.out.println("Will generate RedactionLog for " + files.size() + " files.");
TEST_FILE_ID = "1000";
files.forEach(this::generateAndSaveRedactionLog);
}
@ -257,42 +241,46 @@ public class RulesTest {
/**
* Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
* If RedactionLogs Json does not exist, test will fail.
* Test is ignored, because it's for manual tests.
*/
@Ignore
@IfProfileValue(name = "test-groups", value = "rules-test")
@Test
public void analyseAllFilesAndCompareRedactionLogs() {
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
TEST_FILE_ID = "5000";
files.forEach(this::analyseFileAndCompareRedactionLog);
}
@SneakyThrows
public void generateAndSaveRedactionLog(String fileName) {
private void generateAndSaveRedactionLog(String fileName) {
increaseTestFileId();
if (doesRedactionLogExist(fileName)) {
System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
} else {
generateTestFileId(fileName);
loadNerForTest();
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
AnalyzeRequest request = prepareStorage(fileName);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request);
loadNerForTest();
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnalyzeRequest request = prepareStorage(fileName);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request);
saveRedactionLogAsJson(redactionLog, fileName);
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
saveRedactionLogAsJson(redactionLog, fileName);
}
}
@SneakyThrows
public void analyseFileAndCompareRedactionLog(String fileName) {
increaseTestFileId();
generateTestFileId(fileName);
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
@ -410,9 +398,44 @@ public class RulesTest {
}
private void increaseTestFileId() {
private boolean doesRedactionLogExist(String pdfFileName) {
TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
File pdfFile = new File(pdfFileName);
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
File file = new File(directory, fileName);
return file.exists();
}
@SneakyThrows
private void generateTestFileId(String fileName) {
MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
messageDigest.update(fileName.getBytes());
TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16);
}
@SneakyThrows
private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
File pdfFile = new File(pdfFileName);
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
File dr = new File(directory);
boolean created = dr.mkdirs();
if (created) {
System.out.println("Directory was created");
}
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
File file = new File(directory, fileName);
objectMapper.writeValue(file, redactionLog);
System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
}
@ -466,27 +489,6 @@ public class RulesTest {
}
@SneakyThrows
private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
File pdfFile = new File(pdfFileName);
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
File dr = new File(directory);
boolean created = dr.mkdirs();
if (created) {
System.out.println("Directory was created");
}
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
File file = new File(directory, fileName);
objectMapper.writeValue(file, redactionLog);
System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
}
private void loadDictionaryForTest() {
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
@ -633,9 +635,28 @@ public class RulesTest {
}
private static String loadFromClassPath(String path) {
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
if (resource == null) {
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
}
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
StringBuilder sb = new StringBuilder();
String str;
while ((str = br.readLine()) != null) {
sb.append(str).append("\n");
}
return sb.toString();
} catch (IOException e) {
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
}
}
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
public static class RedactionIntegrationTestConfiguration {
public static class RulesTestConfiguration {
@Bean
public KieContainer kieContainer() {