Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6619 - add logic to ignore found table-cells with height or width < 1.

Browse Source 
Also: Fix the tests and add new segmentation-tests and 1 redaction-integration-test. Renamed the latter to fit maven regexp
This commit is contained in:
Thomas Beyer 2023-04-27 17:49:36 +02:00
parent 6f783a9f00
commit 8490690001
9 changed files with 197 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java
View File

@ -66,4 +66,8 @@ public class Cell extends Rectangle {
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
}
public boolean hasMinimumSize() {
return this.getHeight() >= 1 && this.getWidth() >= 1;
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java
View File

@ -261,7 +261,9 @@ public class Table extends AbstractTextContainer {
if (intersectionCell.isPresent()) {
cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
}
row.add(cell);
if (cell.hasMinimumSize()) {
row.add(cell);
}
}
prevX = x;
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java
View File

@ -88,7 +88,7 @@ public class TableExtractionService {
for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
TextBlock textBlock = (TextBlock) abstractTextContainer;
for (Cell cell : cells) {
if (cell.intersects(textBlock.getPdfMinX(),
if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
textBlock.getPdfMinY(),
textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@ -109,7 +109,7 @@ public class TableExtractionService {
List<Cell> overlappingCells = new ArrayList<>();
for (Cell c : cells) {
if (c.intersects(area)) {
if (c.hasMinimumSize() && c.intersects(area)) {
overlappingCells.add(c);
}
}

39
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTestV2.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -42,8 +42,8 @@ import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTestV2.RedactionIntegrationTestConfiguration.class)
public class RedactionIntegrationTestV2 extends AbstractRedactionIntegrationTest {
@Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class)
public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules_v2.drl");
@ -151,4 +151,39 @@ public class RedactionIntegrationTestV2 extends AbstractRedactionIntegrationTest
}
/**
* The case in this test: The term 'Evans P.G. is very close to a table-cell. It will get redacted nevertheless.
*/
@Test
@SneakyThrows
public void testTermGetsRedactedEvenItsCloseToCellBorder() {
AnalyzeRequest request = uploadFileToStorage("files/SinglePages/VV-931175_Page1.pdf");
dictionary.clear();
falsePositive.clear();
dictionary.put(DICTIONARY_AUTHOR, Arrays.asList("Evans P.G."));
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(1);
RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
assertThat(redactionLogEntry.getValue()).isEqualTo("Evans P.G.");
assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
}
}

182
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java
View File

@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -105,6 +106,14 @@ public class PdfSegmentationServiceTest {
}
@SneakyThrows
private void prepareStorage() {
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
}
@Test
@SneakyThrows
public void testMapping() {
@ -207,12 +216,13 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 4);
validateTable(document, 0, 1, 1, 0, 0);
validateTable(document, 1, 3, 2, 0, 2);
validateTable(document, 2, 9, 20, 0, 180);
validateTable(document, 3, 11, 31, 0, 263);
validateTable(document, 1, 2, 2, 0, 0);
validateTable(document, 2, 7, 20, 0, 140);
validateTable(document, 3, 8, 31, 0, 170);
}
@ -224,9 +234,45 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 1);
validateTable(document, 0, 15, 9, 0, 74);
validateTable(document, 0, 8, 9, 0, 2);
List<List<String>> values = Arrays.asList(
Arrays.asList(
"Annex point Reference within DAR/RAR",
"Author, date",
"Study title",
"Analytical method Author, date, No.",
"Technique, LOQ of the method, validated working range",
"Method meets analytical validation criteria",
"Remarks (in case validation criteria are not met)",
"Acceptability of the method"
),
Arrays.asList(
"",
"Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
"Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
"Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
"Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
"",
"Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
"Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"
),
Arrays.asList(
"CA 7.1.2.1.1 DAR (2009)",
"Evans P.G. 2001 TMJ4569B, VV-323245",
"Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
"Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
"LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
"Y",
"N/A",
"Y"
)
);
validateTable(document, 0, values);
}
@ -238,12 +284,13 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 4);
validateTable(document, 0, 3, 2, 0, 0);
validateTable(document, 1, 4, 2, 0, 2);
validateTable(document, 1, 3, 2, 0, 0);
validateTable(document, 2, 3, 3, 0, 0);
validateTable(document, 3, 4, 3, 0, 3);
validateTable(document, 3, 3, 3, 0, 0);
}
@ -274,12 +321,12 @@ public class PdfSegmentationServiceTest {
validateTableSize(document, 6);
validateTable(document, 0, 3, 1, 0, 1);
validateTable(document, 1, 3, 1, 0, 1);
validateTable(document, 2, 3, 5, 0, 5);
validateTable(document, 3, 3, 5, 0, 5);
validateTable(document, 4, 3, 4, 0, 4);
validateTable(document, 5, 3, 1, 0, 1);
validateTable(document, 0, 2, 1, 0, 0);
validateTable(document, 1, 2, 1, 0, 0);
validateTable(document, 2, 2, 5, 0, 0);
validateTable(document, 3, 2, 5, 0, 0);
validateTable(document, 4, 2, 4, 0, 0);
validateTable(document, 5, 2, 1, 0, 0);
}
@ -310,7 +357,6 @@ public class PdfSegmentationServiceTest {
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 1);
validateTable(document, 0, 9, 9, 0, 0);
}
@ -323,9 +369,10 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 1);
validateTable(document, 0, 10, 5, 6, 5);
validateTable(document, 0, 9, 5, 6, 0);
}
@ -337,6 +384,7 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 1);
validateTable(document, 0, 9, 6, 7, 0);
@ -350,6 +398,7 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 1);
validateTable(document, 0, 10, 6, 0, 1);
@ -363,6 +412,7 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 2);
validateTable(document, 0, 2, 2, 0, 0);
validateTable(document, 1, 1, 1, 0, 0);
@ -378,7 +428,9 @@ public class PdfSegmentationServiceTest {
"files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 2);
validateTable(document, 0, 7, 8, 1, 0);
validateTable(document, 1, 7, 8, 1, 0);
@ -393,9 +445,11 @@ public class PdfSegmentationServiceTest {
"files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 2);
validateTable(document, 0, 4, 17, 0, 0);
validateTable(document, 1, 8, 12, 0, 12);
validateTable(document, 1, 7, 12, 0, 0);
}
@ -407,7 +461,9 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 2);
validateTable(document, 0, 5, 14, 4, 0);
validateTable(document, 1, 7, 12, 0, 0);
@ -421,7 +477,9 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 2);
validateTable(document, 0, 5, 17, 3, 0);
validateTable(document, 1, 5, 16, 2, 0);
@ -435,7 +493,9 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 4);
validateTable(document, 0, 4, 4, 0, 0);
validateTable(document, 1, 1, 1, 0, 0);
validateTable(document, 2, 2, 3, 0, 0);
@ -443,27 +503,94 @@ public class PdfSegmentationServiceTest {
}
@Test
public void testDoc15Page18() throws IOException {
prepareStorage();
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 1);
validateTable(document, 0, 11, 8, 0, 0);
}
@Test
public void testDoc28Page23() throws IOException {
prepareStorage();
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 2);
validateTable(document, 0, 6, 8, 0, 2);
validateTable(document, 1, 6, 8, 0, 1);
}
@Test
public void testDoc24Page17() throws IOException {
prepareStorage();
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 1);
validateTable(document, 0, 9, 5, 2, 0);
}
@Test
public void testDoc30Page5() throws IOException {
prepareStorage();
ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
validateTableSize(document, 1);
validateTable(document, 0, 3, 5, 0, 0);
}
private void validateTable(Document document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {
Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
List<List<Cell>> rows = table.getRows();
int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
int emptyCellsCoundFound = 0;
for (List<Cell> entry : rows) {
for (Cell cell : entry) {
if (cell.toString().equals("")) {
emptyCellsCoundFound++;
}
}
}
assertThat(emptyCellsCoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);
assertThat(emptyCellsFoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);
assertThat(table.getColCount()).isEqualTo(colCount);
assertThat(table.getRowCount()).isEqualTo(rowCount);
}
private void validateTable(Document document, int tableIndex, List<List<String>> values) {
Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
List<List<Cell>> rows = table.getRows();
List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();
List<String> valuesFlattened = values.stream().flatMap(List::stream).toList();
for (int i = 0; i < valuesFlattened.size(); i++) {
Cell cell = rowsFlattened.get(i);
String value = valuesFlattened.get(i);
assertThat(cell.toString()).isEqualTo(value);
}
}
private void validateTableSize(Document document, int tableSize) {
@ -472,12 +599,5 @@ public class PdfSegmentationServiceTest {
}
@SneakyThrows
private void prepareStorage() {
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
}
}

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf Normal file
View File

Binary file not shown.

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf Normal file
View File

Binary file not shown.

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf Normal file
View File

Binary file not shown.

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf Normal file
View File

Binary file not shown.