RED-4510: Rearranged RulesTest.java
This commit is contained in:
Philipp Schramm
parent
40380a12a3
commit
ac0e1cef5e
@ -115,6 +115,7 @@ public class RulesTest {
|
|||||||
private static final String REDACTION_LOG_PATH = RESOURCES_PATH + "RedactionLog/";
|
private static final String REDACTION_LOG_PATH = RESOURCES_PATH + "RedactionLog/";
|
||||||
private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
|
private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
|
||||||
private final static String TEST_DOSSIER_ID = "123";
|
private final static String TEST_DOSSIER_ID = "123";
|
||||||
|
private String TEST_FILE_ID = "123";
|
||||||
private final Map<String, List<String>> dictionary = new HashMap<>();
|
private final Map<String, List<String>> dictionary = new HashMap<>();
|
||||||
private final Map<String, List<String>> dossierDictionary = new HashMap<>();
|
private final Map<String, List<String>> dossierDictionary = new HashMap<>();
|
||||||
private final Map<String, List<String>> falsePositive = new HashMap<>();
|
private final Map<String, List<String>> falsePositive = new HashMap<>();
|
||||||
@ -127,6 +128,7 @@ public class RulesTest {
|
|||||||
private final Colors colors = new Colors();
|
private final Colors colors = new Colors();
|
||||||
private final Map<String, Long> reanalysisVersions = new HashMap<>();
|
private final Map<String, Long> reanalysisVersions = new HashMap<>();
|
||||||
private final Set<String> deleted = new HashSet<>();
|
private final Set<String> deleted = new HashSet<>();
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private RedactionController redactionController;
|
private RedactionController redactionController;
|
||||||
@Autowired
|
@Autowired
|
||||||
@ -151,26 +153,6 @@ public class RulesTest {
|
|||||||
private RabbitTemplate rabbitTemplate;
|
private RabbitTemplate rabbitTemplate;
|
||||||
@MockBean
|
@MockBean
|
||||||
private LegalBasisClient legalBasisClient;
|
private LegalBasisClient legalBasisClient;
|
||||||
private String TEST_FILE_ID = "123";
|
|
||||||
|
|
||||||
|
|
||||||
private static String loadFromClassPath(String path) {
|
|
||||||
|
|
||||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
|
||||||
if (resource == null) {
|
|
||||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
|
||||||
}
|
|
||||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
String str;
|
|
||||||
while ((str = br.readLine()) != null) {
|
|
||||||
sb.append(str).append("\n");
|
|
||||||
}
|
|
||||||
return sb.toString();
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@After
|
@After
|
||||||
@ -201,9 +183,7 @@ public class RulesTest {
|
|||||||
.hexColor("#ffe187")
|
.hexColor("#ffe187")
|
||||||
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
|
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
|
||||||
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
|
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
|
||||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
|
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
|
||||||
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
|
|
||||||
.build()));
|
|
||||||
|
|
||||||
mockDictionaryCalls(null);
|
mockDictionaryCalls(null);
|
||||||
mockDictionaryCalls(0L);
|
mockDictionaryCalls(0L);
|
||||||
@ -212,6 +192,305 @@ public class RulesTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static String loadFromClassPath(String path) {
|
||||||
|
|
||||||
|
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||||
|
if (resource == null) {
|
||||||
|
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||||
|
}
|
||||||
|
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
String str;
|
||||||
|
while ((str = br.readLine()) != null) {
|
||||||
|
sb.append(str).append("\n");
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH
|
||||||
|
*/
|
||||||
|
@Ignore
|
||||||
|
@Test
|
||||||
|
@SneakyThrows
|
||||||
|
public void generateRedactionLogForOneFile() {
|
||||||
|
|
||||||
|
String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
|
||||||
|
generateAndSaveRedactionLog(fileName);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
public void generateAndSaveRedactionLog(String fileName) {
|
||||||
|
|
||||||
|
increaseTestFileId();
|
||||||
|
|
||||||
|
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
|
||||||
|
|
||||||
|
loadNerForTest();
|
||||||
|
|
||||||
|
AnalyzeRequest request = prepareStorage(fileName);
|
||||||
|
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||||
|
analyzeService.analyze(request);
|
||||||
|
|
||||||
|
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
|
saveRedactionLogAsJson(redactionLog, fileName);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void increaseTestFileId() {
|
||||||
|
|
||||||
|
TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private void loadNerForTest() {
|
||||||
|
|
||||||
|
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||||
|
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||||
|
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private AnalyzeRequest prepareStorage(String file) {
|
||||||
|
|
||||||
|
ClassPathResource pdfFileResource = new ClassPathResource(file);
|
||||||
|
|
||||||
|
return prepareStorage(pdfFileResource.getInputStream());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
|
||||||
|
|
||||||
|
File pdfFile = new File(pdfFileName);
|
||||||
|
|
||||||
|
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||||
|
File dr = new File(directory);
|
||||||
|
boolean created = dr.mkdirs();
|
||||||
|
if (created) {
|
||||||
|
System.out.println("Directory was created");
|
||||||
|
}
|
||||||
|
|
||||||
|
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||||
|
File file = new File(directory, fileName);
|
||||||
|
|
||||||
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
|
mapper.registerModule(new JavaTimeModule());
|
||||||
|
mapper.writeValue(file, redactionLog);
|
||||||
|
|
||||||
|
System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private AnalyzeRequest prepareStorage(InputStream stream) {
|
||||||
|
|
||||||
|
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||||
|
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||||
|
.dossierId(TEST_DOSSIER_ID)
|
||||||
|
.fileId(TEST_FILE_ID)
|
||||||
|
.lastProcessed(OffsetDateTime.now())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
var bytes = IOUtils.toByteArray(stream);
|
||||||
|
|
||||||
|
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||||
|
|
||||||
|
return request;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH
|
||||||
|
*/
|
||||||
|
@Ignore
|
||||||
|
@Test
|
||||||
|
@SneakyThrows
|
||||||
|
public void generateRedactionLogForAllFiles() {
|
||||||
|
|
||||||
|
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||||
|
System.out.println("Will generate RedactionLog for " + files.size() + " files.");
|
||||||
|
TEST_FILE_ID = "1000";
|
||||||
|
files.forEach(this::generateAndSaveRedactionLog);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
|
||||||
|
* If RedactionLog Json does not exist, test will fail.
|
||||||
|
*/
|
||||||
|
@Ignore
|
||||||
|
@Test
|
||||||
|
@SneakyThrows
|
||||||
|
public void analyseFileAndCompareRedactionLog() {
|
||||||
|
|
||||||
|
String fileName = "files/Compounds/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf";
|
||||||
|
analyseFileAndCompareRedactionLog(fileName);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
public void analyseFileAndCompareRedactionLog(String fileName) {
|
||||||
|
|
||||||
|
increaseTestFileId();
|
||||||
|
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
|
||||||
|
|
||||||
|
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
|
||||||
|
|
||||||
|
loadNerForTest();
|
||||||
|
|
||||||
|
AnalyzeRequest request = prepareStorage(fileName);
|
||||||
|
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||||
|
analyzeService.analyze(request);
|
||||||
|
|
||||||
|
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
|
assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion());
|
||||||
|
assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber());
|
||||||
|
assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion());
|
||||||
|
assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion());
|
||||||
|
assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
|
||||||
|
assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
|
||||||
|
|
||||||
|
assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
|
||||||
|
assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
|
||||||
|
|
||||||
|
for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
|
||||||
|
var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis()
|
||||||
|
.stream()
|
||||||
|
.filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName()))
|
||||||
|
.filter(lb -> lb.getDescription().equalsIgnoreCase(redactionLegalBasis.getDescription()))
|
||||||
|
.filter(lb -> lb.getReason().equalsIgnoreCase(redactionLegalBasis.getReason()))
|
||||||
|
.findFirst();
|
||||||
|
assertThat(savedRedactionLegalBasis).isPresent();
|
||||||
|
}
|
||||||
|
|
||||||
|
for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
|
||||||
|
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst();
|
||||||
|
assertThat(savedRedactionLogEntry).isPresent();
|
||||||
|
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry());
|
||||||
|
assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId());
|
||||||
|
|
||||||
|
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
||||||
|
var savedRectangle = savedRedactionLogEntry.get()
|
||||||
|
.getPositions()
|
||||||
|
.stream()
|
||||||
|
.filter(r -> r.getPage() == rectangle.getPage())
|
||||||
|
.filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX())
|
||||||
|
.filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY())
|
||||||
|
.filter(r -> r.getHeight() == rectangle.getHeight())
|
||||||
|
.filter(r -> r.getWidth() == rectangle.getWidth())
|
||||||
|
.findFirst();
|
||||||
|
assertThat(savedRectangle).isPresent();
|
||||||
|
}
|
||||||
|
|
||||||
|
for (RedactionLogComment comment : redactionLogEntry.getComments()) {
|
||||||
|
var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst();
|
||||||
|
assertThat(savedComment).isPresent();
|
||||||
|
assertThat(savedComment.get().getId()).isEqualTo(comment.getId());
|
||||||
|
assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser());
|
||||||
|
assertThat(savedComment.get().getText()).isEqualTo(comment.getText());
|
||||||
|
assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId());
|
||||||
|
assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Change change : redactionLogEntry.getChanges()) {
|
||||||
|
var savedChange = savedRedactionLogEntry.get()
|
||||||
|
.getChanges()
|
||||||
|
.stream()
|
||||||
|
.filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
|
||||||
|
.filter(c -> c.getType() == change.getType())
|
||||||
|
.findFirst();
|
||||||
|
assertThat(savedChange).isPresent();
|
||||||
|
}
|
||||||
|
|
||||||
|
for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
|
||||||
|
var savedManualChange = savedRedactionLogEntry.get()
|
||||||
|
.getManualChanges()
|
||||||
|
.stream()
|
||||||
|
.filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
|
||||||
|
.filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType())
|
||||||
|
.filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
|
||||||
|
.filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
|
||||||
|
.findFirst();
|
||||||
|
assertThat(savedManualChange).isPresent();
|
||||||
|
}
|
||||||
|
|
||||||
|
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
|
||||||
|
|
||||||
|
assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
|
||||||
|
assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private RedactionLog loadSavedRedactionLog(String pdfFileName) {
|
||||||
|
|
||||||
|
File pdfFile = new File(pdfFileName);
|
||||||
|
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||||
|
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||||
|
File file = new File(directory, fileName);
|
||||||
|
|
||||||
|
ObjectMapper om = new ObjectMapper();
|
||||||
|
om.registerModule(new JavaTimeModule());
|
||||||
|
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||||
|
|
||||||
|
assertThat(file).exists();
|
||||||
|
return om.readValue(file, RedactionLog.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH
|
||||||
|
* If RedactionLogs Json does not exist, test will fail.
|
||||||
|
*/
|
||||||
|
@Ignore
|
||||||
|
@Test
|
||||||
|
@SneakyThrows
|
||||||
|
public void analyseAllFilesAndCompareRedactionLogs() {
|
||||||
|
|
||||||
|
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||||
|
System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
|
||||||
|
TEST_FILE_ID = "5000";
|
||||||
|
files.forEach(this::analyseFileAndCompareRedactionLog);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
private void loadDictionaryForTest() {
|
private void loadDictionaryForTest() {
|
||||||
|
|
||||||
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
||||||
@ -443,286 +722,6 @@ public class RulesTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH
|
|
||||||
*/
|
|
||||||
@Ignore
|
|
||||||
@Test
|
|
||||||
@SneakyThrows
|
|
||||||
public void generateRedactionLogForOneFile() {
|
|
||||||
|
|
||||||
String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
|
|
||||||
generateRedactionLog(fileName);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
public void generateRedactionLog(String fileName) {
|
|
||||||
|
|
||||||
increaseTestFileId();
|
|
||||||
|
|
||||||
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
|
|
||||||
|
|
||||||
loadNerForTest();
|
|
||||||
|
|
||||||
AnalyzeRequest request = prepareStorage(fileName);
|
|
||||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
|
||||||
analyzeService.analyze(request);
|
|
||||||
|
|
||||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
|
||||||
|
|
||||||
saveRedactionLogAsJson(redactionLog, fileName);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void increaseTestFileId() {
|
|
||||||
|
|
||||||
TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
private void loadNerForTest() {
|
|
||||||
|
|
||||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
|
||||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
|
||||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
private AnalyzeRequest prepareStorage(String file) {
|
|
||||||
|
|
||||||
ClassPathResource pdfFileResource = new ClassPathResource(file);
|
|
||||||
|
|
||||||
return prepareStorage(pdfFileResource.getInputStream());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
|
|
||||||
|
|
||||||
File pdfFile = new File(pdfFileName);
|
|
||||||
|
|
||||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
|
||||||
File dr = new File(directory);
|
|
||||||
boolean created = dr.mkdirs();
|
|
||||||
if (created) {
|
|
||||||
System.out.println("Directory was created");
|
|
||||||
}
|
|
||||||
|
|
||||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
|
||||||
File file = new File(directory, fileName);
|
|
||||||
|
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
|
||||||
mapper.registerModule(new JavaTimeModule());
|
|
||||||
mapper.writeValue(file, redactionLog);
|
|
||||||
|
|
||||||
System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
private AnalyzeRequest prepareStorage(InputStream stream) {
|
|
||||||
|
|
||||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
|
||||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
|
||||||
.dossierId(TEST_DOSSIER_ID)
|
|
||||||
.fileId(TEST_FILE_ID)
|
|
||||||
.lastProcessed(OffsetDateTime.now())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
var bytes = IOUtils.toByteArray(stream);
|
|
||||||
|
|
||||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
|
||||||
|
|
||||||
return request;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH
|
|
||||||
*/
|
|
||||||
@Ignore
|
|
||||||
@Test
|
|
||||||
@SneakyThrows
|
|
||||||
public void generateRedactionLogForAllFiles() {
|
|
||||||
|
|
||||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
|
||||||
System.out.println("Will generate RedactionLog for " + files.size() + " files.");
|
|
||||||
TEST_FILE_ID = "1000";
|
|
||||||
files.forEach(this::generateRedactionLog);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
|
|
||||||
* If RedactionLog Json does not exist, test will fail.
|
|
||||||
*/
|
|
||||||
@Ignore
|
|
||||||
@Test
|
|
||||||
@SneakyThrows
|
|
||||||
public void analyseFileAndCompareRedactionLog() {
|
|
||||||
|
|
||||||
String fileName = "files/Compounds/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf";
|
|
||||||
analyseFileAndCompareRedactionLog(fileName);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
public void analyseFileAndCompareRedactionLog(String fileName) {
|
|
||||||
|
|
||||||
increaseTestFileId();
|
|
||||||
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
|
|
||||||
|
|
||||||
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
|
|
||||||
|
|
||||||
loadNerForTest();
|
|
||||||
|
|
||||||
AnalyzeRequest request = prepareStorage(fileName);
|
|
||||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
|
||||||
analyzeService.analyze(request);
|
|
||||||
|
|
||||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
|
||||||
|
|
||||||
assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion());
|
|
||||||
assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber());
|
|
||||||
assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion());
|
|
||||||
assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion());
|
|
||||||
assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
|
|
||||||
assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
|
|
||||||
|
|
||||||
assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
|
|
||||||
assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
|
|
||||||
|
|
||||||
for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
|
|
||||||
var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis()
|
|
||||||
.stream()
|
|
||||||
.filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName()))
|
|
||||||
.filter(lb -> lb.getDescription().equalsIgnoreCase(redactionLegalBasis.getDescription()))
|
|
||||||
.filter(lb -> lb.getReason().equalsIgnoreCase(redactionLegalBasis.getReason()))
|
|
||||||
.findFirst();
|
|
||||||
assertThat(savedRedactionLegalBasis).isPresent();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
|
|
||||||
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst();
|
|
||||||
assertThat(savedRedactionLogEntry).isPresent();
|
|
||||||
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry());
|
|
||||||
assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId());
|
|
||||||
|
|
||||||
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
|
||||||
var savedRectangle = savedRedactionLogEntry.get()
|
|
||||||
.getPositions()
|
|
||||||
.stream()
|
|
||||||
.filter(r -> r.getPage() == rectangle.getPage())
|
|
||||||
.filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX())
|
|
||||||
.filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY())
|
|
||||||
.filter(r -> r.getHeight() == rectangle.getHeight())
|
|
||||||
.filter(r -> r.getWidth() == rectangle.getWidth())
|
|
||||||
.findFirst();
|
|
||||||
assertThat(savedRectangle).isPresent();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (RedactionLogComment comment : redactionLogEntry.getComments()) {
|
|
||||||
var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst();
|
|
||||||
assertThat(savedComment).isPresent();
|
|
||||||
assertThat(savedComment.get().getId()).isEqualTo(comment.getId());
|
|
||||||
assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser());
|
|
||||||
assertThat(savedComment.get().getText()).isEqualTo(comment.getText());
|
|
||||||
assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId());
|
|
||||||
assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId());
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
for (Change change : redactionLogEntry.getChanges()) {
|
|
||||||
var savedChange = savedRedactionLogEntry.get()
|
|
||||||
.getChanges()
|
|
||||||
.stream()
|
|
||||||
.filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
|
|
||||||
.filter(c -> c.getType() == change.getType())
|
|
||||||
.findFirst();
|
|
||||||
assertThat(savedChange).isPresent();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
|
|
||||||
var savedManualChange = savedRedactionLogEntry.get()
|
|
||||||
.getManualChanges()
|
|
||||||
.stream()
|
|
||||||
.filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
|
|
||||||
.filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType())
|
|
||||||
.filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
|
|
||||||
.filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
|
|
||||||
.findFirst();
|
|
||||||
assertThat(savedManualChange).isPresent();
|
|
||||||
}
|
|
||||||
|
|
||||||
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
|
|
||||||
|
|
||||||
assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
|
|
||||||
assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
private RedactionLog loadSavedRedactionLog(String pdfFileName) {
|
|
||||||
|
|
||||||
File pdfFile = new File(pdfFileName);
|
|
||||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
|
||||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
|
||||||
File file = new File(directory, fileName);
|
|
||||||
|
|
||||||
ObjectMapper om = new ObjectMapper();
|
|
||||||
om.registerModule(new JavaTimeModule());
|
|
||||||
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
||||||
|
|
||||||
assertThat(file).exists();
|
|
||||||
return om.readValue(file, RedactionLog.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH
|
|
||||||
* If RedactionLogs Json does not exist, test will fail.
|
|
||||||
*/
|
|
||||||
@Ignore
|
|
||||||
@Test
|
|
||||||
@SneakyThrows
|
|
||||||
public void analyseAllFilesAndCompareRedactionLogs() {
|
|
||||||
|
|
||||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
|
||||||
System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
|
|
||||||
TEST_FILE_ID = "5000";
|
|
||||||
files.forEach(this::analyseFileAndCompareRedactionLog);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
private Set<String> getFileNames(Set<String> fileNames, Path dir) {
|
private Set<String> getFileNames(Set<String> fileNames, Path dir) {
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user