From a2301fa8ca1e0067edf698eb264d2e179142e1f0 Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Tue, 26 Jul 2022 16:03:47 +0200 Subject: [PATCH] RED-4510: Generated Redaction Log and Generating will not run with profile --- .../redaction/v1/server/RulesTest.java | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index e1884245..be162884 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -49,6 +49,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.core.io.ClassPathResource; +import org.springframework.test.annotation.IfProfileValue; import org.springframework.test.context.junit4.SpringRunner; import com.amazonaws.services.s3.AmazonS3; @@ -118,6 +119,10 @@ public class RulesTest { private final static String TEST_DOSSIER_TEMPLATE_ID = "123"; private final static String TEST_DOSSIER_ID = "123"; private String TEST_FILE_ID = "123"; + + private int counter = 0; + + private int fileSize = 0; private final Map> dictionary = new HashMap<>(); private final Map> dossierDictionary = new HashMap<>(); private final Map> falsePositive = new HashMap<>(); @@ -206,6 +211,7 @@ public class RulesTest { public void generateRedactionLogForOneFile() { String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf"; + fileSize = 1; generateAndSaveRedactionLog(fileName, true); } @@ -220,6 +226,7 @@ public class RulesTest { Set files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH)); System.out.println("Will generate RedactionLog for " + files.size() + " files."); + fileSize = files.size(); files.forEach(this::generateAndSaveRedactionLog); } @@ -231,9 +238,9 @@ public class RulesTest { @Test public void analyseFileAndCompareRedactionLog() { - String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf"; //TODO PSC -// String fileName = "scanned/VV-377031.pdf"; - fileName = "files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf"; +// String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf"; //TODO PSC + String fileName = "scanned/VV-377031.pdf"; + fileSize = 1; analyseFileAndCompareRedactionLog(fileName); } @@ -242,12 +249,14 @@ public class RulesTest { * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH. * If RedactionLog Json for one file does not exist, whole test will fail. */ -// @IfProfileValue(name = "test-groups", value = "rules-test") //TODO PSC + @IfProfileValue(name = "test-groups", value = "rules-test") @Test + @Ignore//TODO PSC public void analyseAllFilesAndCompareRedactionLogs() { Set files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH)); System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs."); + fileSize = files.size(); files.forEach(this::analyseFileAndCompareRedactionLog); } @@ -261,13 +270,14 @@ public class RulesTest { @SneakyThrows private void generateAndSaveRedactionLog(String fileName, boolean force) { + counter++; if (!force && doesRedactionLogExist(fileName)) { - System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists."); + System.out.println("(" + counter + "/" + fileSize + ") Skip generating RedactionLog as Json for " + fileName + " because it already exists."); } else { generateTestFileId(fileName); - System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID); + System.out.println("(" + counter + "/" + fileSize + ") Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID); loadNerForTest(); @@ -286,8 +296,9 @@ public class RulesTest { @SneakyThrows public void analyseFileAndCompareRedactionLog(String fileName) { + counter++; generateTestFileId(fileName); - System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog"); + System.out.println("(" + counter + "/" + fileSize + ") Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog"); RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName); @@ -470,6 +481,25 @@ public class RulesTest { } + @SneakyThrows + private Set getFileNames(Set fileNames, Path dir) { + + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path path : stream) { + if (path.toFile().isDirectory()) { + getFileNames(fileNames, path); + } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) { + String absolutePath = cleanPath(path.toAbsolutePath().toString()); + int pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + RESOURCES_PATH.length(); + + fileNames.add(StringUtils.substring(absolutePath, pos)); + } + } + } + return fileNames; + } + + private String cleanPath(String path) { return StringUtils.replace(path, "\\", "/"); @@ -614,30 +644,6 @@ public class RulesTest { } - @SneakyThrows - private Set getFileNames(Set fileNames, Path dir) { - - try (DirectoryStream stream = Files.newDirectoryStream(dir)) { - for (Path path : stream) { - if (path.toFile().isDirectory()) { - getFileNames(fileNames, path); - } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) { - String absolutePath = cleanPath(path.toAbsolutePath().toString()); - int pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + RESOURCES_PATH.length(); - - if (StringUtils.contains(absolutePath, "scanned/VV-377031.pdf")) { - System.out.println("SKIPPED! " + absolutePath);// TODO PSC - } else { - fileNames.add(StringUtils.substring(absolutePath, pos)); - } - - } - } - } - return fileNames; - } - - private Type getDictionaryResponse(String type, boolean isDossierDictionary) { return Type.builder()