RED-4510: Rearranged RulesTest.java
This commit is contained in:
Philipp Schramm
parent
40380a12a3
commit
ac0e1cef5e
@ -115,6 +115,7 @@ public class RulesTest {
|
||||
private static final String REDACTION_LOG_PATH = RESOURCES_PATH + "RedactionLog/";
|
||||
private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
|
||||
private final static String TEST_DOSSIER_ID = "123";
|
||||
private String TEST_FILE_ID = "123";
|
||||
private final Map<String, List<String>> dictionary = new HashMap<>();
|
||||
private final Map<String, List<String>> dossierDictionary = new HashMap<>();
|
||||
private final Map<String, List<String>> falsePositive = new HashMap<>();
|
||||
@ -127,6 +128,7 @@ public class RulesTest {
|
||||
private final Colors colors = new Colors();
|
||||
private final Map<String, Long> reanalysisVersions = new HashMap<>();
|
||||
private final Set<String> deleted = new HashSet<>();
|
||||
|
||||
@Autowired
|
||||
private RedactionController redactionController;
|
||||
@Autowired
|
||||
@ -151,26 +153,6 @@ public class RulesTest {
|
||||
private RabbitTemplate rabbitTemplate;
|
||||
@MockBean
|
||||
private LegalBasisClient legalBasisClient;
|
||||
private String TEST_FILE_ID = "123";
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@After
|
||||
@ -201,9 +183,7 @@ public class RulesTest {
|
||||
.hexColor("#ffe187")
|
||||
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.build()));
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
|
||||
|
||||
mockDictionaryCalls(null);
|
||||
mockDictionaryCalls(0L);
|
||||
@ -212,6 +192,305 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void generateRedactionLogForOneFile() {
|
||||
|
||||
String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
|
||||
generateAndSaveRedactionLog(fileName);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void generateAndSaveRedactionLog(String fileName) {
|
||||
|
||||
increaseTestFileId();
|
||||
|
||||
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
|
||||
|
||||
loadNerForTest();
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
saveRedactionLogAsJson(redactionLog, fileName);
|
||||
}
|
||||
|
||||
|
||||
private void increaseTestFileId() {
|
||||
|
||||
TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(String file) {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(file);
|
||||
|
||||
return prepareStorage(pdfFileResource.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
|
||||
|
||||
File pdfFile = new File(pdfFileName);
|
||||
|
||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||
File dr = new File(directory);
|
||||
boolean created = dr.mkdirs();
|
||||
if (created) {
|
||||
System.out.println("Directory was created");
|
||||
}
|
||||
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
mapper.registerModule(new JavaTimeModule());
|
||||
mapper.writeValue(file, redactionLog);
|
||||
|
||||
System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(InputStream stream) {
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var bytes = IOUtils.toByteArray(stream);
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||
|
||||
return request;
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void generateRedactionLogForAllFiles() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will generate RedactionLog for " + files.size() + " files.");
|
||||
TEST_FILE_ID = "1000";
|
||||
files.forEach(this::generateAndSaveRedactionLog);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
|
||||
* If RedactionLog Json does not exist, test will fail.
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void analyseFileAndCompareRedactionLog() {
|
||||
|
||||
String fileName = "files/Compounds/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf";
|
||||
analyseFileAndCompareRedactionLog(fileName);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void analyseFileAndCompareRedactionLog(String fileName) {
|
||||
|
||||
increaseTestFileId();
|
||||
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
|
||||
|
||||
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
|
||||
|
||||
loadNerForTest();
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion());
|
||||
assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber());
|
||||
assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion());
|
||||
assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion());
|
||||
assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
|
||||
assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
|
||||
|
||||
assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
|
||||
assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
|
||||
|
||||
for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
|
||||
var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis()
|
||||
.stream()
|
||||
.filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName()))
|
||||
.filter(lb -> lb.getDescription().equalsIgnoreCase(redactionLegalBasis.getDescription()))
|
||||
.filter(lb -> lb.getReason().equalsIgnoreCase(redactionLegalBasis.getReason()))
|
||||
.findFirst();
|
||||
assertThat(savedRedactionLegalBasis).isPresent();
|
||||
}
|
||||
|
||||
for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
|
||||
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst();
|
||||
assertThat(savedRedactionLogEntry).isPresent();
|
||||
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
|
||||
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
|
||||
assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
|
||||
assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());
|
||||
assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule());
|
||||
assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle());
|
||||
assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis());
|
||||
assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported());
|
||||
assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted());
|
||||
assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint());
|
||||
assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation());
|
||||
assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive());
|
||||
assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
|
||||
assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
|
||||
assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
|
||||
assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
|
||||
assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
|
||||
assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
|
||||
assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset());
|
||||
assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage());
|
||||
assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency());
|
||||
assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry());
|
||||
assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry());
|
||||
assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
|
||||
assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId());
|
||||
|
||||
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
||||
var savedRectangle = savedRedactionLogEntry.get()
|
||||
.getPositions()
|
||||
.stream()
|
||||
.filter(r -> r.getPage() == rectangle.getPage())
|
||||
.filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX())
|
||||
.filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY())
|
||||
.filter(r -> r.getHeight() == rectangle.getHeight())
|
||||
.filter(r -> r.getWidth() == rectangle.getWidth())
|
||||
.findFirst();
|
||||
assertThat(savedRectangle).isPresent();
|
||||
}
|
||||
|
||||
for (RedactionLogComment comment : redactionLogEntry.getComments()) {
|
||||
var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst();
|
||||
assertThat(savedComment).isPresent();
|
||||
assertThat(savedComment.get().getId()).isEqualTo(comment.getId());
|
||||
assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser());
|
||||
assertThat(savedComment.get().getText()).isEqualTo(comment.getText());
|
||||
assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId());
|
||||
assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId());
|
||||
|
||||
}
|
||||
|
||||
for (Change change : redactionLogEntry.getChanges()) {
|
||||
var savedChange = savedRedactionLogEntry.get()
|
||||
.getChanges()
|
||||
.stream()
|
||||
.filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
|
||||
.filter(c -> c.getType() == change.getType())
|
||||
.findFirst();
|
||||
assertThat(savedChange).isPresent();
|
||||
}
|
||||
|
||||
for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
|
||||
var savedManualChange = savedRedactionLogEntry.get()
|
||||
.getManualChanges()
|
||||
.stream()
|
||||
.filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
|
||||
.filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType())
|
||||
.filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
|
||||
.filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
|
||||
.findFirst();
|
||||
assertThat(savedManualChange).isPresent();
|
||||
}
|
||||
|
||||
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
|
||||
|
||||
assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
|
||||
assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private RedactionLog loadSavedRedactionLog(String pdfFileName) {
|
||||
|
||||
File pdfFile = new File(pdfFileName);
|
||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
|
||||
ObjectMapper om = new ObjectMapper();
|
||||
om.registerModule(new JavaTimeModule());
|
||||
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
|
||||
assertThat(file).exists();
|
||||
return om.readValue(file, RedactionLog.class);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH
|
||||
* If RedactionLogs Json does not exist, test will fail.
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void analyseAllFilesAndCompareRedactionLogs() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
|
||||
TEST_FILE_ID = "5000";
|
||||
files.forEach(this::analyseFileAndCompareRedactionLog);
|
||||
}
|
||||
|
||||
|
||||
private void loadDictionaryForTest() {
|
||||
|
||||
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
||||
@ -443,286 +722,6 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void generateRedactionLogForOneFile() {
|
||||
|
||||
String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
|
||||
generateRedactionLog(fileName);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void generateRedactionLog(String fileName) {
|
||||
|
||||
increaseTestFileId();
|
||||
|
||||
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
|
||||
|
||||
loadNerForTest();
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
saveRedactionLogAsJson(redactionLog, fileName);
|
||||
}
|
||||
|
||||
|
||||
private void increaseTestFileId() {
|
||||
|
||||
TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(String file) {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(file);
|
||||
|
||||
return prepareStorage(pdfFileResource.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
|
||||
|
||||
File pdfFile = new File(pdfFileName);
|
||||
|
||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||
File dr = new File(directory);
|
||||
boolean created = dr.mkdirs();
|
||||
if (created) {
|
||||
System.out.println("Directory was created");
|
||||
}
|
||||
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
mapper.registerModule(new JavaTimeModule());
|
||||
mapper.writeValue(file, redactionLog);
|
||||
|
||||
System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(InputStream stream) {
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var bytes = IOUtils.toByteArray(stream);
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||
|
||||
return request;
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void generateRedactionLogForAllFiles() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will generate RedactionLog for " + files.size() + " files.");
|
||||
TEST_FILE_ID = "1000";
|
||||
files.forEach(this::generateRedactionLog);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
|
||||
* If RedactionLog Json does not exist, test will fail.
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void analyseFileAndCompareRedactionLog() {
|
||||
|
||||
String fileName = "files/Compounds/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf";
|
||||
analyseFileAndCompareRedactionLog(fileName);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void analyseFileAndCompareRedactionLog(String fileName) {
|
||||
|
||||
increaseTestFileId();
|
||||
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
|
||||
|
||||
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
|
||||
|
||||
loadNerForTest();
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion());
|
||||
assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber());
|
||||
assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion());
|
||||
assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion());
|
||||
assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
|
||||
assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
|
||||
|
||||
assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
|
||||
assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
|
||||
|
||||
for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
|
||||
var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis()
|
||||
.stream()
|
||||
.filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName()))
|
||||
.filter(lb -> lb.getDescription().equalsIgnoreCase(redactionLegalBasis.getDescription()))
|
||||
.filter(lb -> lb.getReason().equalsIgnoreCase(redactionLegalBasis.getReason()))
|
||||
.findFirst();
|
||||
assertThat(savedRedactionLegalBasis).isPresent();
|
||||
}
|
||||
|
||||
for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
|
||||
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst();
|
||||
assertThat(savedRedactionLogEntry).isPresent();
|
||||
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
|
||||
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
|
||||
assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
|
||||
assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());
|
||||
assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule());
|
||||
assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle());
|
||||
assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis());
|
||||
assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported());
|
||||
assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted());
|
||||
assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint());
|
||||
assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation());
|
||||
assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive());
|
||||
assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
|
||||
assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
|
||||
assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
|
||||
assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
|
||||
assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
|
||||
assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
|
||||
assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset());
|
||||
assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage());
|
||||
assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency());
|
||||
assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry());
|
||||
assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry());
|
||||
assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
|
||||
assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId());
|
||||
|
||||
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
||||
var savedRectangle = savedRedactionLogEntry.get()
|
||||
.getPositions()
|
||||
.stream()
|
||||
.filter(r -> r.getPage() == rectangle.getPage())
|
||||
.filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX())
|
||||
.filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY())
|
||||
.filter(r -> r.getHeight() == rectangle.getHeight())
|
||||
.filter(r -> r.getWidth() == rectangle.getWidth())
|
||||
.findFirst();
|
||||
assertThat(savedRectangle).isPresent();
|
||||
}
|
||||
|
||||
for (RedactionLogComment comment : redactionLogEntry.getComments()) {
|
||||
var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst();
|
||||
assertThat(savedComment).isPresent();
|
||||
assertThat(savedComment.get().getId()).isEqualTo(comment.getId());
|
||||
assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser());
|
||||
assertThat(savedComment.get().getText()).isEqualTo(comment.getText());
|
||||
assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId());
|
||||
assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId());
|
||||
|
||||
}
|
||||
|
||||
for (Change change : redactionLogEntry.getChanges()) {
|
||||
var savedChange = savedRedactionLogEntry.get()
|
||||
.getChanges()
|
||||
.stream()
|
||||
.filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
|
||||
.filter(c -> c.getType() == change.getType())
|
||||
.findFirst();
|
||||
assertThat(savedChange).isPresent();
|
||||
}
|
||||
|
||||
for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
|
||||
var savedManualChange = savedRedactionLogEntry.get()
|
||||
.getManualChanges()
|
||||
.stream()
|
||||
.filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
|
||||
.filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType())
|
||||
.filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
|
||||
.filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
|
||||
.findFirst();
|
||||
assertThat(savedManualChange).isPresent();
|
||||
}
|
||||
|
||||
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
|
||||
|
||||
assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
|
||||
assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private RedactionLog loadSavedRedactionLog(String pdfFileName) {
|
||||
|
||||
File pdfFile = new File(pdfFileName);
|
||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
|
||||
ObjectMapper om = new ObjectMapper();
|
||||
om.registerModule(new JavaTimeModule());
|
||||
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
|
||||
assertThat(file).exists();
|
||||
return om.readValue(file, RedactionLog.class);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH
|
||||
* If RedactionLogs Json does not exist, test will fail.
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void analyseAllFilesAndCompareRedactionLogs() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
|
||||
TEST_FILE_ID = "5000";
|
||||
files.forEach(this::analyseFileAndCompareRedactionLog);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private Set<String> getFileNames(Set<String> fileNames, Path dir) {
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user