Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-4510: Generated Redaction Log and Generating will not run with profile

Browse Source
This commit is contained in:
Philipp Schramm 2022-07-26 16:03:47 +02:00
parent e9d925a659
commit a2301fa8ca
1 changed files with 37 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -49,6 +49,7 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.test.annotation.IfProfileValue;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
@ -118,6 +119,10 @@ public class RulesTest {
private final static String TEST_DOSSIER_TEMPLATE_ID = "123"; private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
private final static String TEST_DOSSIER_ID = "123"; private final static String TEST_DOSSIER_ID = "123";
private String TEST_FILE_ID = "123"; private String TEST_FILE_ID = "123";
private int counter = 0;
private int fileSize = 0;
private final Map<String, List<String>> dictionary = new HashMap<>(); private final Map<String, List<String>> dictionary = new HashMap<>();
private final Map<String, List<String>> dossierDictionary = new HashMap<>(); private final Map<String, List<String>> dossierDictionary = new HashMap<>();
private final Map<String, List<String>> falsePositive = new HashMap<>(); private final Map<String, List<String>> falsePositive = new HashMap<>();
@ -206,6 +211,7 @@ public class RulesTest {
public void generateRedactionLogForOneFile() { public void generateRedactionLogForOneFile() {
String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf"; String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
fileSize = 1;
generateAndSaveRedactionLog(fileName, true); generateAndSaveRedactionLog(fileName, true);
} }
@ -220,6 +226,7 @@ public class RulesTest {
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH)); Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
System.out.println("Will generate RedactionLog for " + files.size() + " files."); System.out.println("Will generate RedactionLog for " + files.size() + " files.");
fileSize = files.size();
files.forEach(this::generateAndSaveRedactionLog); files.forEach(this::generateAndSaveRedactionLog);
} }
@ -231,9 +238,9 @@ public class RulesTest {
@Test @Test
public void analyseFileAndCompareRedactionLog() { public void analyseFileAndCompareRedactionLog() {
String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf"; //TODO PSC // String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf"; //TODO PSC
// String fileName = "scanned/VV-377031.pdf"; String fileName = "scanned/VV-377031.pdf";
fileName = "files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf"; fileSize = 1;
analyseFileAndCompareRedactionLog(fileName); analyseFileAndCompareRedactionLog(fileName);
} }
@ -242,12 +249,14 @@ public class RulesTest {
* Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH. * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
* If RedactionLog Json for one file does not exist, whole test will fail. * If RedactionLog Json for one file does not exist, whole test will fail.
*/ */
// @IfProfileValue(name = "test-groups", value = "rules-test") //TODO PSC @IfProfileValue(name = "test-groups", value = "rules-test")
@Test @Test
@Ignore//TODO PSC
public void analyseAllFilesAndCompareRedactionLogs() { public void analyseAllFilesAndCompareRedactionLogs() {
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH)); Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs."); System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
fileSize = files.size();
files.forEach(this::analyseFileAndCompareRedactionLog); files.forEach(this::analyseFileAndCompareRedactionLog);
} }
@ -261,13 +270,14 @@ public class RulesTest {
@SneakyThrows @SneakyThrows
private void generateAndSaveRedactionLog(String fileName, boolean force) { private void generateAndSaveRedactionLog(String fileName, boolean force) {
counter++;
if (!force && doesRedactionLogExist(fileName)) { if (!force && doesRedactionLogExist(fileName)) {
System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists."); System.out.println("(" + counter + "/" + fileSize + ") Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
} else { } else {
generateTestFileId(fileName); generateTestFileId(fileName);
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID); System.out.println("(" + counter + "/" + fileSize + ") Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
loadNerForTest(); loadNerForTest();
@ -286,8 +296,9 @@ public class RulesTest {
@SneakyThrows @SneakyThrows
public void analyseFileAndCompareRedactionLog(String fileName) { public void analyseFileAndCompareRedactionLog(String fileName) {
counter++;
generateTestFileId(fileName); generateTestFileId(fileName);
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog"); System.out.println("(" + counter + "/" + fileSize + ") Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName); RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
@ -470,6 +481,25 @@ public class RulesTest {
} }
@SneakyThrows
private Set<String> getFileNames(Set<String> fileNames, Path dir) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
for (Path path : stream) {
if (path.toFile().isDirectory()) {
getFileNames(fileNames, path);
} else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
String absolutePath = cleanPath(path.toAbsolutePath().toString());
int pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + RESOURCES_PATH.length();
fileNames.add(StringUtils.substring(absolutePath, pos));
}
}
}
return fileNames;
}
private String cleanPath(String path) { private String cleanPath(String path) {
return StringUtils.replace(path, "\\", "/"); return StringUtils.replace(path, "\\", "/");
@ -614,30 +644,6 @@ public class RulesTest {
} }
@SneakyThrows
private Set<String> getFileNames(Set<String> fileNames, Path dir) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
for (Path path : stream) {
if (path.toFile().isDirectory()) {
getFileNames(fileNames, path);
} else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
String absolutePath = cleanPath(path.toAbsolutePath().toString());
int pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + RESOURCES_PATH.length();
if (StringUtils.contains(absolutePath, "scanned/VV-377031.pdf")) {
System.out.println("SKIPPED! " + absolutePath);// TODO PSC
} else {
fileNames.add(StringUtils.substring(absolutePath, pos));
}
}
}
}
return fileNames;
}
private Type getDictionaryResponse(String type, boolean isDossierDictionary) { private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
return Type.builder() return Type.builder()