Compare commits

...
Author SHA1 Message Date
Thierry Goeckel 6483e637c6 Pull request #32: Remove '-' in headernames
Merge in RED/redaction-service from tablesHeader3 to master

* commit '272c7cc2287b146664b8bb277c3731262ac78a19':
  Remove '-' in headernames
2020-08-24 16:15:44 +02:00
deiflaender 272c7cc228 Remove '-' in headernames 2020-08-24 15:39:50 +02:00
Thierry Goeckel 5586531c33 Pull request #31: Made rules for table more stabil and flexible
Merge in RED/redaction-service from tablesRules2 to master

* commit '559c42154276ea861c09cf29b71d9bb9c17e6e87':
  Made rules for table more stabil and flexible
2020-08-24 14:52:29 +02:00
deiflaender 559c421542 Made rules for table more stabil and flexible 2020-08-24 14:45:30 +02:00
Dominique Eiflaender d026f4e1db Pull request #30: Fix test and logic of same page table merge for one row tables
Merge in RED/redaction-service from bugfix/fix-test-table-merge to master

* commit '38ddb1d9c8dbf0e83d40ec916f1ebeb6d0f86f42':
  Fix test and logic of same page table merge for one row tables
2020-08-24 14:22:41 +02:00
Thierry Göckel 38ddb1d9c8 Fix test and logic of same page table merge for one row tables 2020-08-24 14:06:34 +02:00
Thierry Goeckel 92f0af9d21 Pull request #28: Highlight N in vertebrate study tables and fixed rule order
Merge in RED/redaction-service from tableRules to master

* commit 'afa26895c8e78f261ec3700c515f412d8cf605ce':
  Fixed test problem
  Fixed pmd
  Highlight N in vertebrate study tables and fixed rule order
2020-08-24 13:15:11 +02:00
Dominique Eiflaender b24922aa60 Pull request #29: Fix multi-page table merge
Merge in RED/redaction-service from bugfix/multi-page-table-merge to master

* commit 'baa703928f50e53a66f925fcf72082df188482a1':
  Fix multi-page table merge
2020-08-24 13:09:43 +02:00
deiflaender afa26895c8 Fixed test problem 2020-08-24 13:05:23 +02:00
Thierry Göckel baa703928f Fix multi-page table merge 2020-08-24 13:03:03 +02:00
deiflaender 85667c91e4 Fixed pmd 2020-08-24 12:36:49 +02:00
deiflaender 8a471d0771 Highlight N in vertebrate study tables and fixed rule order 2020-08-24 12:24:48 +02:00
Dominique Eiflaender bb1112d0d7 Pull request #27: Fix merging of tables with single header row table
Merge in RED/redaction-service from bugfix/merge-single-row-tables to master

* commit '00a960ee2308e7ed7da23ebd3e85f5ef9db14610':
  Fix merging of tables with single header row table
2020-08-24 12:10:53 +02:00
Thierry Göckel 00a960ee23 Fix merging of tables with single header row table 2020-08-24 11:43:19 +02:00
Dominique Eiflaender 8c08bb3664 Pull request #25: RED-101 & RED-102: Make table structure detection more robust
Merge in RED/redaction-service from bugfix/RED-101 to master

* commit '76369f13f8d41154f3dd3690af81b9567ca1133e':
  Remove redundant comment
  RED-101: Add features as requested in PR
  Add unit test for table structure requirements
  Make table structure detection more robust
2020-08-24 11:09:17 +02:00
Cheng Zhu 14b4f4ab8a Pull request #26: Feature/RED-116: replace gin4-commons with red-commons
Merge in RED/redaction-service from feature/RED-116 to master

* commit '4206e708a20a862083f03d27a992e8661c44abc2':
  RED-116: replace gin4-commons with red-commons
  RED-116: replace gin4-commons with red-commons
2020-08-20 17:22:36 +02:00
cheng 4206e708a2 RED-116: replace gin4-commons with red-commons 2020-08-20 16:52:56 +02:00
cheng c4164b57fb RED-116: replace gin4-commons with red-commons 2020-08-20 16:51:02 +02:00
Thierry Göckel 76369f13f8 Remove redundant comment 2020-08-18 20:53:41 +02:00
deiflaender 89ef8234d2 Added missing NoArgsConstructor to RedactionLogEntry, needed for Json Serialization 2020-08-14 11:21:11 +02:00
14 changed files with 191 additions and 83 deletions
+12 -12
View File
@@ -5,7 +5,7 @@
<parent>
<artifactId>platform-dependency</artifactId>
<groupId>com.iqser.red</groupId>
<version>1.0.1</version>
<version>1.0.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -32,21 +32,21 @@
<dependency>
<groupId>com.iqser.red</groupId>
<artifactId>platform-commons-dependency</artifactId>
<version>1.0.0</version>
<version>1.1.0</version>
<scope>import</scope>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox-tools</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox-tools</artifactId>
<version>${pdfbox.version}</version>
</dependency>
</dependencies>
@@ -3,11 +3,15 @@ package com.iqser.red.service.redaction.v1.model;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RedactionLogEntry {
private String id;
@@ -63,15 +63,15 @@
</dependency>
<!-- commons -->
<dependency>
<groupId>com.iqser.gin4.commons</groupId>
<groupId>com.iqser.red.commons</groupId>
<artifactId>spring-commons</artifactId>
</dependency>
<dependency>
<groupId>com.iqser.gin4.commons</groupId>
<groupId>com.iqser.red.commons</groupId>
<artifactId>logging-commons</artifactId>
</dependency>
<dependency>
<groupId>com.iqser.gin4.commons</groupId>
<groupId>com.iqser.red.commons</groupId>
<artifactId>metric-commons</artifactId>
</dependency>
<!-- other external -->
@@ -104,7 +104,7 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.iqser.gin4.commons</groupId>
<groupId>com.iqser.red.commons</groupId>
<artifactId>test-commons</artifactId>
<scope>test</scope>
</dependency>
@@ -20,7 +20,7 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
@@ -2,13 +2,13 @@ package com.iqser.red.service.redaction.v1.server.controller;
import java.time.OffsetDateTime;
import com.iqser.red.commons.spring.ErrorMessage;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import com.iqser.gin4.commons.api.errorhandling.ErrorMessage;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import lombok.extern.slf4j.Slf4j;
@@ -35,17 +35,12 @@ public class Section {
private Map<String, TextBlock> tabularData;
public boolean isVertebrateStudy() {
return tabularData != null
&& tabularData.containsKey("Vertebrate study Y/N")
&& tabularData.get("Vertebrate study Y/N").getText().equals("Y");
}
public boolean rowEquals(String headerName, String value){
String cleanHeaderName = headerName.replaceAll("\n", "")
.replaceAll(" ", "")
.replaceAll("-", "");
public boolean isNotVertebrateStudy() {
return tabularData != null
&& tabularData.containsKey("Vertebrate study Y/N")
&& tabularData.get("Vertebrate study Y/N").getText().equals("N");
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).getText().equals(value);
}
@@ -176,13 +171,17 @@ public class Section {
}
public void highlightCell(String cellHeader, int ruleNumber) {
public void highlightCell(String cellHeader, int ruleNumber, String type) {
TextBlock value = tabularData.get(cellHeader);
String cleanHeaderName = cellHeader.replaceAll("\n", "")
.replaceAll(" ", "")
.replaceAll("-", "");
TextBlock value = tabularData.get(cleanHeaderName);
if (value == null) {
log.warn("Could not find any data for {}.", cellHeader);
} else {
Entity entity = new Entity(value.getText(), "must_redact", 0, value.getText().length(), headline, sectionNumber);
Entity entity = new Entity(value.getText(), type, 0, value.getText().length(), headline, sectionNumber);
entity.setRedaction(false);
entity.setMatchedRule(ruleNumber);
entity.setRedactionReason(cellHeader);
@@ -192,4 +191,4 @@ public class Section {
}
}
}
@@ -60,9 +60,13 @@ public class EntityRedactionService {
}
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber);
cell.getHeaderCells().forEach(headerCell -> {
String headerName = headerCell.getTextBlocks().get(0).getText()
.replaceAll("\n", " ")
.replaceAll(" ", " ");
StringBuilder headerBuilder = new StringBuilder();
headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
String headerName = headerBuilder.toString()
.replaceAll("\n", "")
.replaceAll(" ", "")
.replaceAll("-", "");
tabularData.put(headerName, cell.getTextBlocks().get(0));
});
for (TextBlock textBlock : cell.getTextBlocks()) {
@@ -4,6 +4,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
@@ -39,22 +40,27 @@ public class SectionsBuilderService {
current.setPage(page.getPageNumber());
if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline, previousTable);
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
chunkBlock.setHeadline(lastHeadline);
lastHeadline = current.getText();
if (CollectionUtils.isNotEmpty(chunkBlock.getTables())) {
previousTable = chunkBlock.getTables().get(0);
}
chunkBlockList.add(chunkBlock);
chunkWords = new ArrayList<>();
if (CollectionUtils.isNotEmpty(chunkBlock.getTables())) {
previousTable = chunkBlock.getTables().get(chunkBlock.getTables().size() - 1);
}
}
if (current instanceof Table) {
Table table = (Table) current;
// Distribute header information for subsequent tables
mergeTableMetadata(table, previousTable);
previousTable = table;
}
chunkWords.add(current);
prev = current;
}
}
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline, previousTable);
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
chunkBlock.setHeadline(lastHeadline);
chunkBlockList.add(chunkBlock);
@@ -62,7 +68,39 @@ public class SectionsBuilderService {
}
private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline, Table previousTable) {
private void mergeTableMetadata(Table currentTable, Table previousTable) {
// Distribute header information for subsequent tables
if (previousTable != null && hasInvalidHeaderInformation(currentTable) && hasValidHeaderInformation(previousTable)) {
List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
// Allow merging of tables if header row is separated from first logical non-header row
if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows()
.get(0)
.size() == tableNonHeaderRow.size()) {
previousTableNonHeaderRow = previousTable.getRows().get(0).stream().map(cell -> {
Cell fakeCell = new Cell(cell.getPoints()[0], cell.getPoints()[2]);
fakeCell.setHeaderCells(Collections.singletonList(cell));
return fakeCell;
}).collect(Collectors.toList());
}
if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
for (int i = currentTable.getRows()
.size() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
List<Cell> row = currentTable.getRows().get(i);
if (row.size() == tableNonHeaderRow.size() && row.stream()
.allMatch(cell -> cell.getHeaderCells().isEmpty())) {
for (int j = 0; j < row.size(); j++) {
row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
}
}
}
}
}
}
private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
Paragraph paragraph = new Paragraph();
TextBlock textBlock = null;
@@ -85,22 +123,6 @@ public class SectionsBuilderService {
} else {
table.setHeadline("Table in: " + lastHeadline);
}
// Distribute header information for subsequent tables
if (previousTable != null && hasInvalidHeaderInformation(table) && hasValidHeaderInformation(previousTable)) {
List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(table);
if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
for (int i = table.getRows().size() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
List<Cell> row = table.getRows().get(i);
if (row.size() == previousTableNonHeaderRow.size()
&& row.stream().allMatch(cell -> cell.getHeaderCells().isEmpty())) {
for (int j = 0; j < row.size(); j++) {
row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
}
}
}
}
}
if (textBlock != null && !alreadyAdded) {
paragraph.getPageBlocks().add(textBlock);
@@ -8,12 +8,6 @@ spring:
profiles:
active: kubernetes
platform.multi-tenancy:
enabled: ${multitenancy.enabled:false}
tenantFilter:
urlPatterns: /redact
urlPatternsToIgnore:
management:
endpoint:
metrics.enabled: ${monitoring.enabled:false}
@@ -214,7 +214,7 @@ public class EntityRedactionServiceTest {
entityRedactionService.processDocument(classifiedDoc, null);
assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8);
assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(4); // FIXME including one false positive "Y"
assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(4);
}
}
@@ -228,11 +228,11 @@ public class EntityRedactionServiceTest {
"global Section section\n" +
"rule \"9: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study\"\n" +
" when\n" +
" Section(isVertebrateStudy())\n" +
" Section(rowEquals(\"Vertebrate study Y/N\", \"Y\"))\n" +
" then\n" +
" section.redact(\"name\", 9, \"Redacted because row is a vertebrate study\");\n" +
" section.redact(\"address\", 9, \"Redacted because rows is a vertebrate study\");\n" +
" section.highlightCell(\"Vertebrate study Y/N\", 9);\n" +
" section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" +
" end";
when(rulesClient.getVersion()).thenReturn(1L);
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));
@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.segmentation;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@@ -18,6 +19,7 @@ import org.springframework.test.context.junit4.SpringRunner;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
@@ -64,4 +66,84 @@ public class PdfSegmentationServiceTest {
}
}
@Test
public void testTableExtraction() throws IOException {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document document = pdfSegmentationService.parseDocument(pdDocument);
assertThat(document.getParagraphs()
.stream()
.flatMap(paragraph -> paragraph.getTables().stream())
.collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getParagraphs()
.stream()
.flatMap(paragraph -> paragraph.getTables().stream())
.collect(Collectors.toList())
.get(0);
assertThat(firstTable.getColCount()).isEqualTo(8);
assertThat(firstTable.getRowCount()).isEqualTo(1);
Table secondTable = document.getParagraphs()
.stream()
.flatMap(paragraph -> paragraph.getTables().stream())
.collect(Collectors.toList())
.get(1);
assertThat(secondTable.getColCount()).isEqualTo(8);
assertThat(secondTable.getRowCount()).isEqualTo(2);
List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
.get(0)
.stream()
.map(Collections::singletonList)
.collect(Collectors.toList());
assertThat(secondTable.getRows().stream()
.allMatch(row -> row.stream()
.map(Cell::getHeaderCells)
.collect(Collectors.toList())
.equals(firstTableHeaderCells)))
.isTrue();
}
}
@Test
public void testMultiPageMetadataPropagation() throws IOException {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document document = pdfSegmentationService.parseDocument(pdDocument);
assertThat(document.getParagraphs()
.stream()
.flatMap(paragraph -> paragraph.getTables().stream())
.collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getParagraphs()
.stream()
.flatMap(paragraph -> paragraph.getTables().stream())
.collect(Collectors.toList())
.get(0);
assertThat(firstTable.getColCount()).isEqualTo(9);
assertThat(firstTable.getRowCount()).isEqualTo(5);
Table secondTable = document.getParagraphs()
.stream()
.flatMap(paragraph -> paragraph.getTables().stream())
.collect(Collectors.toList())
.get(1);
assertThat(secondTable.getColCount()).isEqualTo(9);
assertThat(secondTable.getRowCount()).isEqualTo(6);
List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
.get(firstTable.getRowCount() - 1)
.stream()
.map(Cell::getHeaderCells)
.collect(Collectors.toList());
assertThat(secondTable.getRows().stream()
.allMatch(row -> row.stream()
.map(Cell::getHeaderCells)
.collect(Collectors.toList())
.equals(firstTableHeaderCells)))
.isTrue();
}
}
}
@@ -90,28 +90,31 @@ rule "7: Redact contact information, if Producer is found"
section.redactBetween("No:", "Fax", "address", 7, "Producer was found");
end
rule "8: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study"
when
Section(isVertebrateStudy())
then
section.redact("name", 8, "Redacted because row is a vertebrate study");
section.redact("address", 8, "Redacted because row is a vertebrate study");
section.highlightCell("Vertebrate study Y/N", 9);
end
rule "9: Not redacted because Vertebrate Study = N"
rule "8: Not redacted because Vertebrate Study = N"
when
Section(isNotVertebrateStudy())
Section(rowEquals("Vertebrate study Y/N", "N"))
then
section.redactNot("name", 9, "Not redacted because row is not a vertebrate study");
section.redactNot("address", 9, "Not redacted because row is not a vertebrate study");
section.redactNot("name", 8, "Not redacted because row is not a vertebrate study");
section.redactNot("address", 8, "Not redacted because row is not a vertebrate study");
section.highlightCell("Vertebrate study Y/N", 8, "hint_only");
end
rule "10: Redact if must redact entry is found"
rule "9: Redact if must redact entry is found"
when
eval(section.contains("must_redact")==true);
then
section.redact("name", 10, "must_redact entry was found.");
section.redact("address", 10, "must_redact entry was found.");
end
section.redact("name", 9, "must_redact entry was found.");
section.redact("address", 9, "must_redact entry was found.");
end
rule "10: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study"
when
Section(rowEquals("Vertebrate study Y/N", "Y"))
then
section.redact("name", 10, "Redacted because row is a vertebrate study");
section.redact("address", 10, "Redacted because row is a vertebrate study");
section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
end