RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist
This commit is contained in:
Philipp Schramm
parent
58e16fe39a
commit
78368579b3
@ -1,10 +1,12 @@
|
||||
package buildjob;
|
||||
|
||||
import java.time.DayOfWeek;
|
||||
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
|
||||
|
||||
import java.time.LocalTime;
|
||||
|
||||
import com.atlassian.bamboo.specs.api.BambooSpec;
|
||||
import com.atlassian.bamboo.specs.api.builders.BambooKey;
|
||||
import com.atlassian.bamboo.specs.api.builders.Variable;
|
||||
import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
|
||||
import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
|
||||
import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
|
||||
@ -17,20 +19,16 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
|
||||
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
|
||||
import com.atlassian.bamboo.specs.api.builders.project.Project;
|
||||
import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
|
||||
import com.atlassian.bamboo.specs.api.builders.Variable;
|
||||
import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
|
||||
import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
|
||||
import com.atlassian.bamboo.specs.util.BambooServer;
|
||||
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
|
||||
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
|
||||
|
||||
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
|
||||
import com.atlassian.bamboo.specs.util.BambooServer;
|
||||
|
||||
/**
|
||||
* Plan configuration for Bamboo.
|
||||
@ -57,6 +55,11 @@ public class PlanSpec {
|
||||
PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
|
||||
bambooServer.publish(planPermission);
|
||||
|
||||
Plan nightPlan = new PlanSpec().createNightPlan();
|
||||
bambooServer.publish(nightPlan);
|
||||
PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
|
||||
bambooServer.publish(nightPlanPermission);
|
||||
|
||||
Plan secPlan = new PlanSpec().createSecBuild();
|
||||
bambooServer.publish(secPlan);
|
||||
PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
|
||||
@ -127,22 +130,38 @@ public class PlanSpec {
|
||||
.linkedRepositories("RED / " + SERVICE_NAME)
|
||||
|
||||
.triggers(new BitbucketServerTrigger())
|
||||
.planBranchManagement(new PlanBranchManagement()
|
||||
.createForVcsBranch()
|
||||
.delete(new BranchCleanup()
|
||||
.whenInactiveInRepositoryAfterDays(14))
|
||||
.planBranchManagement(new PlanBranchManagement().createForVcsBranch()
|
||||
.delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
|
||||
.notificationForCommitters());
|
||||
}
|
||||
|
||||
|
||||
public Plan createNightPlan() {
|
||||
|
||||
return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
|
||||
.variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
|
||||
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
|
||||
.enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
|
||||
.cleanCheckout(true)
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
|
||||
.argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
|
||||
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
|
||||
.enabled(true))
|
||||
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
.linkedRepositories("RED / " + SERVICE_NAME)
|
||||
.triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
|
||||
.planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
|
||||
}
|
||||
|
||||
|
||||
public Plan createSecBuild() {
|
||||
return new Plan(
|
||||
project(),
|
||||
SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
|
||||
.description("Security Analysis Plan")
|
||||
.stages(new Stage("Default Stage")
|
||||
.jobs(new Job("Default Job",
|
||||
new BambooKey("JOB1"))
|
||||
.tasks(
|
||||
|
||||
return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
|
||||
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(
|
||||
new ScriptTask()
|
||||
.description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
@ -174,4 +193,5 @@ public class PlanSpec {
|
||||
.createForVcsBranchMatching("release.*")
|
||||
.notificationForCommitters());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
|
||||
public class PlanSpecTest {
|
||||
@Test
|
||||
public void checkYourPlanOffline() throws PropertiesValidationException {
|
||||
|
||||
Plan plan = new PlanSpec().createPlan();
|
||||
EntityPropertiesBuilders.build(plan);
|
||||
|
||||
Plan nightPlan = new PlanSpec().createNightPlan();
|
||||
EntityPropertiesBuilders.build(nightPlan);
|
||||
|
||||
Plan secPlan = new PlanSpec().createSecBuild();
|
||||
EntityPropertiesBuilders.build(secPlan);
|
||||
}
|
||||
|
||||
@ -10,12 +10,14 @@ import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.math.BigInteger;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
@ -48,6 +50,7 @@ import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.annotation.IfProfileValue;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
@ -86,7 +89,7 @@ import lombok.SneakyThrows;
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@Import(RulesTest.RedactionIntegrationTestConfiguration.class)
|
||||
@Import(RulesTest.RulesTestConfiguration.class)
|
||||
public class RulesTest {
|
||||
|
||||
private static final String RULES_PATH = "drools/testRules.drl";
|
||||
@ -151,34 +154,6 @@ public class RulesTest {
|
||||
private LegalBasisClient legalBasisClient;
|
||||
|
||||
|
||||
@After
|
||||
public void cleanupStorage() {
|
||||
|
||||
if (this.storageService instanceof FileSystemBackedStorageService) {
|
||||
((FileSystemBackedStorageService) this.storageService).clearStorage();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Before
|
||||
public void stubClients() {
|
||||
|
||||
@ -201,9 +176,7 @@ public class RulesTest {
|
||||
.hexColor("#ffe187")
|
||||
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.build()));
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
|
||||
|
||||
mockDictionaryCalls(null);
|
||||
mockDictionaryCalls(0L);
|
||||
@ -212,8 +185,28 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
|
||||
* If the RedactionLog already exists, the generating will be skipped
|
||||
* Test is ignored, because it's for manual tests.
|
||||
*/
|
||||
@Ignore
|
||||
@ -225,18 +218,12 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
|
||||
* Test is ignored, because it's for manual tests.
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
public void generateRedactionLogForAllFiles() {
|
||||
@After
|
||||
public void cleanupStorage() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will generate RedactionLog for " + files.size() + " files.");
|
||||
TEST_FILE_ID = "1000";
|
||||
files.forEach(this::generateAndSaveRedactionLog);
|
||||
if (this.storageService instanceof FileSystemBackedStorageService) {
|
||||
((FileSystemBackedStorageService) this.storageService).clearStorage();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -254,165 +241,76 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
|
||||
* If a RedactionLog already exists, the generating will be skipped
|
||||
*/
|
||||
@IfProfileValue(name = "test-groups", value = "rules-test")
|
||||
@Test
|
||||
public void generateRedactionLogForAllFiles() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will generate RedactionLog for " + files.size() + " files.");
|
||||
files.forEach(this::generateAndSaveRedactionLog);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void generateAndSaveRedactionLog(String fileName) {
|
||||
|
||||
if (doesRedactionLogExist(fileName)) {
|
||||
System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
|
||||
|
||||
} else {
|
||||
generateTestFileId(fileName);
|
||||
|
||||
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
|
||||
|
||||
loadNerForTest();
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
saveRedactionLogAsJson(redactionLog, fileName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
|
||||
* If RedactionLogs Json does not exist, test will fail.
|
||||
* Test is ignored, because it's for manual tests.
|
||||
*/
|
||||
@Ignore
|
||||
@IfProfileValue(name = "test-groups", value = "rules-test")
|
||||
@Test
|
||||
public void analyseAllFilesAndCompareRedactionLogs() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
|
||||
TEST_FILE_ID = "5000";
|
||||
files.forEach(this::analyseFileAndCompareRedactionLog);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void generateAndSaveRedactionLog(String fileName) {
|
||||
private boolean doesRedactionLogExist(String pdfFileName) {
|
||||
|
||||
increaseTestFileId();
|
||||
|
||||
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
|
||||
|
||||
loadNerForTest();
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
saveRedactionLogAsJson(redactionLog, fileName);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void analyseFileAndCompareRedactionLog(String fileName) {
|
||||
|
||||
increaseTestFileId();
|
||||
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
|
||||
|
||||
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
|
||||
|
||||
loadNerForTest();
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
// All timestamps are ignored, because they are for sure different
|
||||
assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion());
|
||||
assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber());
|
||||
assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion());
|
||||
assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion());
|
||||
assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
|
||||
assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
|
||||
|
||||
assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
|
||||
assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
|
||||
|
||||
for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
|
||||
var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis()
|
||||
.stream()
|
||||
.filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName()))
|
||||
.filter(lb -> lb.getDescription().equalsIgnoreCase(redactionLegalBasis.getDescription()))
|
||||
.filter(lb -> lb.getReason().equalsIgnoreCase(redactionLegalBasis.getReason()))
|
||||
.findFirst();
|
||||
assertThat(savedRedactionLegalBasis).isPresent();
|
||||
}
|
||||
|
||||
for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
|
||||
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst();
|
||||
assertThat(savedRedactionLogEntry).isPresent();
|
||||
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
|
||||
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
|
||||
assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
|
||||
assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());
|
||||
assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule());
|
||||
assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle());
|
||||
assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis());
|
||||
assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported());
|
||||
assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted());
|
||||
assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint());
|
||||
assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation());
|
||||
assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive());
|
||||
assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
|
||||
assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
|
||||
assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
|
||||
assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
|
||||
assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
|
||||
assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
|
||||
assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset());
|
||||
assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage());
|
||||
assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency());
|
||||
assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry());
|
||||
assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry());
|
||||
assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
|
||||
assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId());
|
||||
|
||||
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
||||
var savedRectangle = savedRedactionLogEntry.get()
|
||||
.getPositions()
|
||||
.stream()
|
||||
.filter(r -> r.getPage() == rectangle.getPage())
|
||||
.filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX())
|
||||
.filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY())
|
||||
.filter(r -> r.getHeight() == rectangle.getHeight())
|
||||
.filter(r -> r.getWidth() == rectangle.getWidth())
|
||||
.findFirst();
|
||||
assertThat(savedRectangle).isPresent();
|
||||
}
|
||||
|
||||
for (RedactionLogComment comment : redactionLogEntry.getComments()) {
|
||||
var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst();
|
||||
assertThat(savedComment).isPresent();
|
||||
assertThat(savedComment.get().getId()).isEqualTo(comment.getId());
|
||||
assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser());
|
||||
assertThat(savedComment.get().getText()).isEqualTo(comment.getText());
|
||||
assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId());
|
||||
assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId());
|
||||
|
||||
}
|
||||
|
||||
for (Change change : redactionLogEntry.getChanges()) {
|
||||
var savedChange = savedRedactionLogEntry.get()
|
||||
.getChanges()
|
||||
.stream()
|
||||
.filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
|
||||
.filter(c -> c.getType() == change.getType())
|
||||
.findFirst();
|
||||
assertThat(savedChange).isPresent();
|
||||
}
|
||||
|
||||
for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
|
||||
var savedManualChange = savedRedactionLogEntry.get()
|
||||
.getManualChanges()
|
||||
.stream()
|
||||
.filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
|
||||
.filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType())
|
||||
.filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
|
||||
.filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
|
||||
.findFirst();
|
||||
assertThat(savedManualChange).isPresent();
|
||||
}
|
||||
|
||||
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
|
||||
|
||||
assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
|
||||
assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
|
||||
}
|
||||
File pdfFile = new File(pdfFileName);
|
||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
return file.exists();
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void increaseTestFileId() {
|
||||
@SneakyThrows
|
||||
private void generateTestFileId(String fileName) {
|
||||
|
||||
TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
|
||||
MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
|
||||
messageDigest.update(fileName.getBytes());
|
||||
TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16);
|
||||
}
|
||||
|
||||
|
||||
@ -633,9 +531,130 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void analyseFileAndCompareRedactionLog(String fileName) {
|
||||
|
||||
generateTestFileId(fileName);
|
||||
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
|
||||
|
||||
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
|
||||
|
||||
loadNerForTest();
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
// All timestamps are ignored, because they are for sure different
|
||||
assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion());
|
||||
assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber());
|
||||
assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion());
|
||||
assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion());
|
||||
assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
|
||||
assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
|
||||
|
||||
assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
|
||||
assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
|
||||
|
||||
for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
|
||||
var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis()
|
||||
.stream()
|
||||
.filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName()))
|
||||
.filter(lb -> lb.getDescription().equalsIgnoreCase(redactionLegalBasis.getDescription()))
|
||||
.filter(lb -> lb.getReason().equalsIgnoreCase(redactionLegalBasis.getReason()))
|
||||
.findFirst();
|
||||
assertThat(savedRedactionLegalBasis).isPresent();
|
||||
}
|
||||
|
||||
for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
|
||||
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst();
|
||||
assertThat(savedRedactionLogEntry).isPresent();
|
||||
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
|
||||
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
|
||||
assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
|
||||
assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());
|
||||
assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule());
|
||||
assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle());
|
||||
assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis());
|
||||
assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported());
|
||||
assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted());
|
||||
assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint());
|
||||
assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation());
|
||||
assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive());
|
||||
assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
|
||||
assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
|
||||
assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
|
||||
assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
|
||||
assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
|
||||
assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
|
||||
assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset());
|
||||
assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage());
|
||||
assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency());
|
||||
assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry());
|
||||
assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry());
|
||||
assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
|
||||
assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId());
|
||||
|
||||
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
||||
var savedRectangle = savedRedactionLogEntry.get()
|
||||
.getPositions()
|
||||
.stream()
|
||||
.filter(r -> r.getPage() == rectangle.getPage())
|
||||
.filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX())
|
||||
.filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY())
|
||||
.filter(r -> r.getHeight() == rectangle.getHeight())
|
||||
.filter(r -> r.getWidth() == rectangle.getWidth())
|
||||
.findFirst();
|
||||
assertThat(savedRectangle).isPresent();
|
||||
}
|
||||
|
||||
for (RedactionLogComment comment : redactionLogEntry.getComments()) {
|
||||
var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst();
|
||||
assertThat(savedComment).isPresent();
|
||||
assertThat(savedComment.get().getId()).isEqualTo(comment.getId());
|
||||
assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser());
|
||||
assertThat(savedComment.get().getText()).isEqualTo(comment.getText());
|
||||
assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId());
|
||||
assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId());
|
||||
|
||||
}
|
||||
|
||||
for (Change change : redactionLogEntry.getChanges()) {
|
||||
var savedChange = savedRedactionLogEntry.get()
|
||||
.getChanges()
|
||||
.stream()
|
||||
.filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
|
||||
.filter(c -> c.getType() == change.getType())
|
||||
.findFirst();
|
||||
assertThat(savedChange).isPresent();
|
||||
}
|
||||
|
||||
for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
|
||||
var savedManualChange = savedRedactionLogEntry.get()
|
||||
.getManualChanges()
|
||||
.stream()
|
||||
.filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
|
||||
.filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType())
|
||||
.filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
|
||||
.filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
|
||||
.findFirst();
|
||||
assertThat(savedManualChange).isPresent();
|
||||
}
|
||||
|
||||
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
|
||||
|
||||
assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
|
||||
assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
|
||||
public static class RedactionIntegrationTestConfiguration {
|
||||
public static class RulesTestConfiguration {
|
||||
|
||||
@Bean
|
||||
public KieContainer kieContainer() {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user