RED-8701 - Move files to customer data repositories
This commit is contained in:
parent
ac4f51652f
commit
5f637dcbca
@ -51,7 +51,7 @@ public class AnalysisTest extends AbstractRedactionIntegrationTest {
|
|||||||
@Disabled
|
@Disabled
|
||||||
public void analyzeTableDemoFile() throws IOException {
|
public void analyzeTableDemoFile() throws IOException {
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
|
||||||
|
|
||||||
System.out.println("Start Full integration test");
|
System.out.println("Start Full integration test");
|
||||||
analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
|
analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
|
||||||
|
|||||||
@ -55,7 +55,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
|
|||||||
// @Disabled
|
// @Disabled
|
||||||
public void titleExtraction() throws IOException {
|
public void titleExtraction() throws IOException {
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
|
||||||
|
|
||||||
ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
|
ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
|
||||||
"files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
|
"files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
|
||||||
@ -114,7 +114,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
|
|||||||
// @Disabled
|
// @Disabled
|
||||||
public void testTopOfPage13InNotHeader() throws IOException {
|
public void testTopOfPage13InNotHeader() throws IOException {
|
||||||
|
|
||||||
// Fix In BodyTextFrameService destroys header detection in files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf
|
// Fix In BodyTextFrameService destroys header detection in files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf
|
||||||
// TODO unify logic
|
// TODO unify logic
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf");
|
||||||
|
|||||||
@ -128,7 +128,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
|
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
|
||||||
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
|
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
|
||||||
System.out.println("Start Full integration test");
|
System.out.println("Start Full integration test");
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
System.out.println("Finished structure analysis");
|
System.out.println("Finished structure analysis");
|
||||||
@ -194,7 +194,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
|
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
|
||||||
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
|
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
|
||||||
System.out.println("Start Full integration test");
|
System.out.println("Start Full integration test");
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
System.out.println("Finished structure analysis");
|
System.out.println("Finished structure analysis");
|
||||||
|
|||||||
@ -140,7 +140,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
@Test
|
@Test
|
||||||
public void test270Rotated() {
|
public void test270Rotated() {
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/270Rotated.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SinglePages/270Rotated_Page1_Page2_Amended Residue analytical method for the determ.pdf");
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
AnalyzeResult result = analyzeService.analyze(request);
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
assertThat(result).isNotNull();
|
assertThat(result).isNotNull();
|
||||||
@ -150,7 +150,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testFludioxonilDuplicatedImageEntries() {
|
public void testFludioxonilDuplicatedImageEntries() {
|
||||||
|
|
||||||
AnalyzeRequest request = prepareStorage("files/Fludioxonil/Fludioxonil_duplicates.pdf",
|
AnalyzeRequest request = prepareStorage("files/syngenta/CustomerFiles/Fludioxonil_duplicates.pdf",
|
||||||
"files/cv_service_empty_response.json",
|
"files/cv_service_empty_response.json",
|
||||||
"files/image_info_fludioxonil_duplicates.json");
|
"files/image_info_fludioxonil_duplicates.json");
|
||||||
|
|
||||||
@ -176,7 +176,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
|
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/merge_images.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SinglePages/merge_images - Page241_18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
AnalyzeResult result = analyzeService.analyze(request);
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
|
|
||||||
@ -293,7 +293,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
|
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
|
||||||
importedRedactionClasspathResource.getInputStream());
|
importedRedactionClasspathResource.getInputStream());
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
|
||||||
System.out.println("Start Full integration test");
|
System.out.println("Start Full integration test");
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
System.out.println("Finished structure analysis");
|
System.out.println("Finished structure analysis");
|
||||||
@ -337,7 +337,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testAddFileAttribute() {
|
public void testAddFileAttribute() {
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
|
||||||
|
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
AnalyzeResult result = analyzeService.analyze(request);
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
@ -445,7 +445,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
// TODO: this is already broken on master, no idea how to fix it. Most likely more responses need to be stubbed.
|
// TODO: this is already broken on master, no idea how to fix it. Most likely more responses need to be stubbed.
|
||||||
public void redactionTestSeparatedRedaction() throws IOException {
|
public void redactionTestSeparatedRedaction() throws IOException {
|
||||||
|
|
||||||
String fileName = "files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights (1) (1).pdf";
|
String fileName = "files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf";
|
||||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf";
|
String outputFileName = OsUtils.getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf";
|
||||||
|
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
@ -555,7 +555,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testChangeComputation() throws IOException {
|
public void testChangeComputation() throws IOException {
|
||||||
|
|
||||||
String fileName = "files/new/test1S1T1.pdf";
|
String fileName = "files/syngenta/CustomerFiles/SinglePages/test1S1T1.pdf";
|
||||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage(fileName);
|
AnalyzeRequest request = uploadFileToStorage(fileName);
|
||||||
@ -725,7 +725,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
};
|
};
|
||||||
List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
|
List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/new/PublishedInformationTest.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/new/PublishedInformationTest_Page214_19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017.pdf");
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
manualRedactions.getIdsToRemove()
|
manualRedactions.getIdsToRemove()
|
||||||
@ -784,7 +784,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
System.out.println("testTableRedaction");
|
System.out.println("testTableRedaction");
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
AnalyzeResult result = analyzeService.analyze(request);
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
|
|
||||||
@ -805,7 +805,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
|
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
|
||||||
String fileName = "files/new/VV-511309.pdf";
|
String fileName = "files/syngenta/CustomerFiles/VV-511309.pdf";
|
||||||
String tableServiceResponseFile = "files/cv_table_response_VV-511309.json";
|
String tableServiceResponseFile = "files/cv_table_response_VV-511309.json";
|
||||||
|
|
||||||
AnalyzeRequest request = prepareStorage(fileName, tableServiceResponseFile);
|
AnalyzeRequest request = prepareStorage(fileName, tableServiceResponseFile);
|
||||||
@ -829,7 +829,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
|
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
|
||||||
String fileName = "files/new/VV-511309_OCR.pdf";
|
String fileName = "files/syngenta/CustomerFiles/VV-511309_OCR.pdf";
|
||||||
String tableServiceResponseFile = "files/cv_table_response_VV-511309.json";
|
String tableServiceResponseFile = "files/cv_table_response_VV-511309.json";
|
||||||
|
|
||||||
AnalyzeRequest request = prepareStorage(fileName, tableServiceResponseFile);
|
AnalyzeRequest request = prepareStorage(fileName, tableServiceResponseFile);
|
||||||
@ -958,7 +958,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
|
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/new/S157.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SinglePages/S157_S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
|
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
|
||||||
@ -1004,7 +1004,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
@Disabled // TODO Figure out why this changed
|
@Disabled // TODO Figure out why this changed
|
||||||
public void phantomCellsDocumentTest() {
|
public void phantomCellsDocumentTest() {
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/Phantom Cells.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/Phantom Cells - Page10_92 Trinexapac-ethyl_RAR_02_Volume_2_2017-12-04.pdf");
|
||||||
|
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
AnalyzeResult result = analyzeService.analyze(request);
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
@ -1029,7 +1029,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
|
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/sponsor_companies.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SinglePages/sponsor_companies - Page4_83 Pydiflumetofen_DAR_02_Volume_2_2017-07-26.pdf");
|
||||||
|
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
AnalyzeResult result = analyzeService.analyze(request);
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
@ -1050,7 +1050,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
@Disabled
|
@Disabled
|
||||||
public void resizeRedactionTest() throws IOException {
|
public void resizeRedactionTest() throws IOException {
|
||||||
|
|
||||||
String pdfFile = "files/Minimal Examples/Single Table.pdf";
|
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/Single Table - Page87_S-Metolachlor_RAR_07_Volume_3CA_B-5_2018-09-06.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
|
||||||
@ -1163,7 +1163,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
System.out.println("expandByRegex");
|
System.out.println("expandByRegex");
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
AnalyzeResult result = analyzeService.analyze(request);
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
|
|
||||||
@ -1349,7 +1349,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testRemovingAndAddingRedactionOnTheSameValue() {
|
public void testRemovingAndAddingRedactionOnTheSameValue() {
|
||||||
|
|
||||||
String pdfFile = "files/new/test1S1T1.pdf";
|
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/test1S1T1.pdf";
|
||||||
|
|
||||||
String manualAddId = UUID.randomUUID().toString();
|
String manualAddId = UUID.randomUUID().toString();
|
||||||
String manualAddId2 = UUID.randomUUID().toString();
|
String manualAddId2 = UUID.randomUUID().toString();
|
||||||
@ -1571,7 +1571,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
|||||||
@Disabled
|
@Disabled
|
||||||
public void testPydiflumetofenMongo() throws IOException {
|
public void testPydiflumetofenMongo() throws IOException {
|
||||||
|
|
||||||
String fileName = "files/Pydiflumetofen/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26.pdf";
|
String fileName = "files/syngenta/CustomerFiles/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26.pdf";
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage(fileName);
|
AnalyzeRequest request = uploadFileToStorage(fileName);
|
||||||
|
|
||||||
|
|||||||
@ -149,7 +149,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testTermGetsRedactedEvenItsCloseToCellBorder() {
|
public void testTermGetsRedactedEvenItsCloseToCellBorder() {
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage("files/SinglePages/VV-931175_Page1.pdf");
|
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SinglePages/Page1_VV-931175.pdf");
|
||||||
|
|
||||||
dictionary.clear();
|
dictionary.clear();
|
||||||
falsePositive.clear();
|
falsePositive.clear();
|
||||||
|
|||||||
@ -308,7 +308,7 @@ public class RulesTest {
|
|||||||
@Test
|
@Test
|
||||||
public void generateRedactionLogForOneFile() {
|
public void generateRedactionLogForOneFile() {
|
||||||
|
|
||||||
String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
|
String fileName = "files/syngenta/CustomerFiles/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
|
||||||
fileSize = 1;
|
fileSize = 1;
|
||||||
generateAndSaveRedactionLog(fileName, true);
|
generateAndSaveRedactionLog(fileName, true);
|
||||||
}
|
}
|
||||||
@ -338,7 +338,7 @@ public class RulesTest {
|
|||||||
@Test
|
@Test
|
||||||
public void analyseFileAndCompareRedactionLog() {
|
public void analyseFileAndCompareRedactionLog() {
|
||||||
|
|
||||||
String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf";
|
String fileName = "files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf";
|
||||||
fileSize = 1;
|
fileSize = 1;
|
||||||
analyseFileAndCompareRedactionLog(fileName);
|
analyseFileAndCompareRedactionLog(fileName);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -55,8 +55,8 @@ public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTe
|
|||||||
fileNameWithPdf = filename + ".pdf";
|
fileNameWithPdf = filename + ".pdf";
|
||||||
}
|
}
|
||||||
AnalyzeRequest request;
|
AnalyzeRequest request;
|
||||||
if (fileNameWithPdf.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
|
if (fileNameWithPdf.equals("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
|
||||||
request = prepareStorage(fileNameWithPdf, "files/cv_service_empty_response.json", fileNameWithPdf.replace(".pdf", ".IMAGE_INFO.json"));
|
request = prepareStorage(fileNameWithPdf, "files/cv_service_empty_response.json", "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json");
|
||||||
} else {
|
} else {
|
||||||
request = uploadFileToStorage(fileNameWithPdf);
|
request = uploadFileToStorage(fileNameWithPdf);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -53,7 +53,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
|
|||||||
@Test
|
@Test
|
||||||
public void assertEntitiesAreDuplicatedWithTheirTableCell() {
|
public void assertEntitiesAreDuplicatedWithTheirTableCell() {
|
||||||
|
|
||||||
Document document = buildGraph("files/Minimal Examples/Meto1_Page22.pdf");
|
Document document = buildGraph("files/syngenta/CustomerFiles/SinglePages/Meto1_Page22_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
|
||||||
List<TextEntity> entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document)
|
List<TextEntity> entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document)
|
||||||
.toList();
|
.toList();
|
||||||
assertEquals(3, entities.size());
|
assertEquals(3, entities.size());
|
||||||
@ -209,7 +209,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
|
|||||||
@Test
|
@Test
|
||||||
public void findAndCheckMultipleSearchTermsMetolachlor() {
|
public void findAndCheckMultipleSearchTermsMetolachlor() {
|
||||||
|
|
||||||
Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
Document document = buildGraph("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
||||||
assertValueAndPageAndIntersectingNodes(document, "sideeffects", 4);
|
assertValueAndPageAndIntersectingNodes(document, "sideeffects", 4);
|
||||||
assertValueAndPageAndIntersectingNodes(document, "Commission Regulation", 9);
|
assertValueAndPageAndIntersectingNodes(document, "Commission Regulation", 9);
|
||||||
assertValueAndPageAndIntersectingNodes(document, "Pre-emergence", 15);
|
assertValueAndPageAndIntersectingNodes(document, "Pre-emergence", 15);
|
||||||
@ -222,7 +222,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
|
|||||||
@Test
|
@Test
|
||||||
public void assertTableStructureMetolachlor() {
|
public void assertTableStructureMetolachlor() {
|
||||||
|
|
||||||
Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
Document document = buildGraph("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
||||||
Table table = (Table) document.getDocumentTree().allEntriesInOrder()
|
Table table = (Table) document.getDocumentTree().allEntriesInOrder()
|
||||||
.filter(entry -> entry.getNode().getPages()
|
.filter(entry -> entry.getNode().getPages()
|
||||||
.stream()
|
.stream()
|
||||||
@ -249,7 +249,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
|
|||||||
@Test
|
@Test
|
||||||
public void assertTextBeforeAndTextAfterForParagraphMetolachlor() {
|
public void assertTextBeforeAndTextAfterForParagraphMetolachlor() {
|
||||||
|
|
||||||
Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
Document document = buildGraph("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
||||||
String searchTerm = "Cucurbit";
|
String searchTerm = "Cucurbit";
|
||||||
TextEntity textEntity = createAndInsertEntity(document, searchTerm);
|
TextEntity textEntity = createAndInsertEntity(document, searchTerm);
|
||||||
|
|
||||||
@ -271,7 +271,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
|
|||||||
@Test
|
@Test
|
||||||
public void assertTextBeforeAndTextAfterForHeadlineMetolachlor() {
|
public void assertTextBeforeAndTextAfterForHeadlineMetolachlor() {
|
||||||
|
|
||||||
Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
Document document = buildGraph("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
||||||
String searchTerm = "of subject matter";
|
String searchTerm = "of subject matter";
|
||||||
int start = document.getTextBlock().indexOf(searchTerm);
|
int start = document.getTextBlock().indexOf(searchTerm);
|
||||||
assert start != -1;
|
assert start != -1;
|
||||||
@ -301,7 +301,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
|
|||||||
@Test
|
@Test
|
||||||
public void assertTextBeforeAndTextAfterForTableCellMetolachlor() {
|
public void assertTextBeforeAndTextAfterForTableCellMetolachlor() {
|
||||||
|
|
||||||
Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
Document document = buildGraph("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
||||||
String searchTerm = "N-deacetylation product";
|
String searchTerm = "N-deacetylation product";
|
||||||
TextEntity textEntity = createAndInsertEntity(document, searchTerm);
|
TextEntity textEntity = createAndInsertEntity(document, searchTerm);
|
||||||
|
|
||||||
|
|||||||
@ -132,7 +132,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testDroolsOnDocumentGraph() {
|
public void testDroolsOnDocumentGraph() {
|
||||||
|
|
||||||
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
String filename = "files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
||||||
Document document = buildGraph(filename);
|
Document document = buildGraph(filename);
|
||||||
|
|
||||||
dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
|
dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
|
||||||
@ -172,7 +172,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
|
|||||||
public void testBuildTextBlockPerformance() {
|
public void testBuildTextBlockPerformance() {
|
||||||
|
|
||||||
int n = 10000;
|
int n = 10000;
|
||||||
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
String filename = "files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
||||||
Document document = buildGraph(filename);
|
Document document = buildGraph(filename);
|
||||||
var start = System.currentTimeMillis();
|
var start = System.currentTimeMillis();
|
||||||
for (int i = 0; i < n; i++) {
|
for (int i = 0; i < n; i++) {
|
||||||
@ -204,7 +204,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testDictionarySearchOnDocumentGraph() {
|
public void testDictionarySearchOnDocumentGraph() {
|
||||||
|
|
||||||
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
String filename = "files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
||||||
Document document = buildGraph(filename);
|
Document document = buildGraph(filename);
|
||||||
|
|
||||||
dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
|
dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
|
||||||
|
|||||||
@ -14,7 +14,7 @@ public class DocumentTableIntegrationTest extends BuildDocumentIntegrationTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testAllTableCellAccessesCorrect() {
|
public void testAllTableCellAccessesCorrect() {
|
||||||
|
|
||||||
Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06");
|
Document document = buildGraph("files/syngenta/CustomerFiles/S-Metolachlor_RAR_02_Volume_2_2018-09-06");
|
||||||
Page pageFive = document.getPages()
|
Page pageFive = document.getPages()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(pageNode -> pageNode.getNumber() == 5)
|
.filter(pageNode -> pageNode.getNumber() == 5)
|
||||||
|
|||||||
@ -23,7 +23,7 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testSearchImplementationWithSingleEntry() {
|
public void testSearchImplementationWithSingleEntry() {
|
||||||
|
|
||||||
Document document = buildGraph("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections");
|
Document document = buildGraph("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections");
|
||||||
|
|
||||||
SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true);
|
SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true);
|
||||||
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
|
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
|
||||||
|
|||||||
@ -233,7 +233,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
System.out.println("testManualRedaction");
|
System.out.println("testManualRedaction");
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
String pdfFile = "files/Minimal Examples/Single Table.pdf";
|
String pdfFile = "files/syngenta/CustomeFiles/SinglePages/Single Table - Page87_S-Metolachlor_RAR_07_Volume_3CA_B-5_2018-09-06.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
|
||||||
@ -291,7 +291,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
System.out.println("testManualRedaction");
|
System.out.println("testManualRedaction");
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
String pdfFile = "files/Minimal Examples/Meto1_Page22.pdf";
|
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/Meto1_Page22_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
|
||||||
|
|||||||
@ -103,7 +103,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
|
|||||||
|
|
||||||
private DocumentAndEntity createNotFoundManualRedaction() {
|
private DocumentAndEntity createNotFoundManualRedaction() {
|
||||||
|
|
||||||
Document document = buildGraph("files/new/VV-919901.pdf");
|
Document document = buildGraph("files/syngenta/CustomerFiles/VV-919901.pdf");
|
||||||
// This is important due to PDFTron Web Viewer reordering the content, such that this string is selectable.
|
// This is important due to PDFTron Web Viewer reordering the content, such that this string is selectable.
|
||||||
String value = "To: Syngenta Ltd. Jealott’s Hill";
|
String value = "To: Syngenta Ltd. Jealott’s Hill";
|
||||||
String type = DICTIONARY_AUTHOR;
|
String type = DICTIONARY_AUTHOR;
|
||||||
@ -149,7 +149,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
|
|||||||
|
|
||||||
private DocumentAndEntity createFoundManualRedaction() {
|
private DocumentAndEntity createFoundManualRedaction() {
|
||||||
|
|
||||||
Document document = buildGraph("files/new/VV-919901.pdf");
|
Document document = buildGraph("files/syngenta/CustomerFiles/VV-919901.pdf");
|
||||||
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
|
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
|
||||||
|
|
||||||
List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document)
|
List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document)
|
||||||
|
|||||||
@ -104,7 +104,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testGetNerEntitiesLargeFile() {
|
public void testGetNerEntitiesLargeFile() {
|
||||||
|
|
||||||
String filePath = "files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf";
|
String filePath = "files/syngenta/CustomerFiles/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf";
|
||||||
String nerEntitiesFilePath = "ner_entities/S-Metolachlor_RAR_02_Volume_2_2018-09-06.NER_ENTITIES.json";
|
String nerEntitiesFilePath = "ner_entities/S-Metolachlor_RAR_02_Volume_2_2018-09-06.NER_ENTITIES.json";
|
||||||
Document document = buildGraphNoImages(filePath);
|
Document document = buildGraphNoImages(filePath);
|
||||||
log.info("Graph built");
|
log.info("Graph built");
|
||||||
|
|||||||
@ -122,7 +122,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testManualSurroundingText() {
|
public void testManualSurroundingText() {
|
||||||
|
|
||||||
String pdfFile = "files/new/S4.pdf";
|
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/S4_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
|
||||||
@ -229,7 +229,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
|||||||
@Test
|
@Test
|
||||||
void testMultipleManualResizeRequests() {
|
void testMultipleManualResizeRequests() {
|
||||||
|
|
||||||
String pdfFile = "files/new/S4.pdf";
|
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/S4_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
|
||||||
@ -344,7 +344,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
|||||||
@Test
|
@Test
|
||||||
void testManualResizeIfMultipleValuesInTheSameNode() {
|
void testManualResizeIfMultipleValuesInTheSameNode() {
|
||||||
|
|
||||||
String pdfFile = "files/new/S4.pdf";
|
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/S4_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
var aoelId = UUID.randomUUID().toString();
|
var aoelId = UUID.randomUUID().toString();
|
||||||
@ -394,7 +394,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
|||||||
@Test
|
@Test
|
||||||
void testManualResizeInADifferentSection() {
|
void testManualResizeInADifferentSection() {
|
||||||
|
|
||||||
String pdfFile = "files/new/S4.pdf";
|
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/S4_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
var aoelId = UUID.randomUUID().toString();
|
var aoelId = UUID.randomUUID().toString();
|
||||||
@ -444,7 +444,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
|||||||
@Test
|
@Test
|
||||||
void testManualResizeNotFound() {
|
void testManualResizeNotFound() {
|
||||||
|
|
||||||
String pdfFile = "files/new/S4.pdf";
|
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/S4_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
var aoelId = UUID.randomUUID().toString();
|
var aoelId = UUID.randomUUID().toString();
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user