Pull request #28: Highlight N in vertebrate study tables and fixed rule order

Merge in RED/redaction-service from tableRules to master

* commit 'afa26895c8e78f261ec3700c515f412d8cf605ce':
  Fixed test problem
  Fixed pmd
  Highlight N in vertebrate study tables and fixed rule order

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.0.1</version>
+        <version>1.0.2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -32,21 +32,21 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.0.0</version>
+                <version>1.1.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>
 
-                <dependency>
-                    <groupId>org.apache.pdfbox</groupId>
-                    <artifactId>pdfbox</artifactId>
-                    <version>${pdfbox.version}</version>
-                </dependency>
-                <dependency>
-                    <groupId>org.apache.pdfbox</groupId>
-                    <artifactId>pdfbox-tools</artifactId>
-                    <version>${pdfbox.version}</version>
-                </dependency>
+            <dependency>
+                <groupId>org.apache.pdfbox</groupId>
+                <artifactId>pdfbox</artifactId>
+                <version>${pdfbox.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.pdfbox</groupId>
+                <artifactId>pdfbox-tools</artifactId>
+                <version>${pdfbox.version}</version>
+            </dependency>
 
         </dependencies>
 

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -3,11 +3,15 @@ package com.iqser.red.service.redaction.v1.model;
 import java.util.ArrayList;
 import java.util.List;
 
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class RedactionLogEntry {
 
     private String id;

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -63,15 +63,15 @@
         </dependency>
         <!-- commons -->
         <dependency>
-            <groupId>com.iqser.gin4.commons</groupId>
+            <groupId>com.iqser.red.commons</groupId>
             <artifactId>spring-commons</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.iqser.gin4.commons</groupId>
+            <groupId>com.iqser.red.commons</groupId>
             <artifactId>logging-commons</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.iqser.gin4.commons</groupId>
+            <groupId>com.iqser.red.commons</groupId>
             <artifactId>metric-commons</artifactId>
         </dependency>
         <!-- other external -->
@@ -104,7 +104,7 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.iqser.gin4.commons</groupId>
+            <groupId>com.iqser.red.commons</groupId>
             <artifactId>test-commons</artifactId>
             <scope>test</scope>
         </dependency>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -20,7 +20,7 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
-import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/ControllerAdvice.java

@@ -2,13 +2,13 @@ package com.iqser.red.service.redaction.v1.server.controller;
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.commons.spring.ErrorMessage;
 import org.springframework.http.HttpStatus;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestControllerAdvice;
 
-import com.iqser.gin4.commons.api.errorhandling.ErrorMessage;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 
 import lombok.extern.slf4j.Slf4j;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -37,15 +37,19 @@ public class Section {
 
     public boolean isVertebrateStudy() {
         return tabularData != null
-                && tabularData.containsKey("Vertebrate study Y/N")
-                && tabularData.get("Vertebrate study Y/N").getText().equals("Y");
+                && (tabularData.containsKey("Vertebrate study Y/N")
+                && tabularData.get("Vertebrate study Y/N").getText().equals("Y")
+                || tabularData.containsKey("Verte brate study Y/N")
+                && tabularData.get("Verte brate study Y/N").getText().equals("Y"));
     }
 
 
     public boolean isNotVertebrateStudy() {
         return tabularData != null
-                && tabularData.containsKey("Vertebrate study Y/N")
-                && tabularData.get("Vertebrate study Y/N").getText().equals("N");
+                && (tabularData.containsKey("Vertebrate study Y/N")
+                && tabularData.get("Vertebrate study Y/N").getText().equals("N")
+                || tabularData.containsKey("Verte brate study Y/N")
+                && tabularData.get("Verte brate study Y/N").getText().equals("N"));
     }
 
 
@@ -176,13 +180,13 @@ public class Section {
     }
 
 
-    public void highlightCell(String cellHeader, int ruleNumber) {
+    public void highlightCell(String cellHeader, int ruleNumber, String type) {
 
         TextBlock value = tabularData.get(cellHeader);
         if (value == null) {
             log.warn("Could not find any data for {}.", cellHeader);
         } else {
-            Entity entity = new Entity(value.getText(), "must_redact", 0, value.getText().length(), headline, sectionNumber);
+            Entity entity = new Entity(value.getText(), type, 0, value.getText().length(), headline, sectionNumber);
             entity.setRedaction(false);
             entity.setMatchedRule(ruleNumber);
             entity.setRedactionReason(cellHeader);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -42,13 +42,17 @@ public class SectionsBuilderService {
                     Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline, previousTable);
                     chunkBlock.setHeadline(lastHeadline);
                     lastHeadline = current.getText();
-                    if (CollectionUtils.isNotEmpty(chunkBlock.getTables())) {
-                        previousTable = chunkBlock.getTables().get(0);
-                    }
                     chunkBlockList.add(chunkBlock);
                     chunkWords = new ArrayList<>();
+                    if (CollectionUtils.isNotEmpty(chunkBlock.getTables())) {
+                        previousTable = chunkBlock.getTables().get(chunkBlock.getTables().size() - 1);
+                    }
+                } else if (current instanceof Table) {
+                    Table table = (Table) current;
+                    // Distribute header information for subsequent tables
+                    mergeTableMetadata(table, previousTable);
+                    previousTable = table;
                 }
-
                 chunkWords.add(current);
                 prev = current;
             }
@@ -62,6 +66,34 @@ public class SectionsBuilderService {
     }
 
 
+    private void mergeTableMetadata(Table currentTable, Table previousTable) {
+
+        // Distribute header information for subsequent tables
+        if (previousTable != null && hasInvalidHeaderInformation(currentTable) && hasValidHeaderInformation(previousTable)) {
+            List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
+            List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
+            // Allow merging of tables if header row is separated from first logical non-header row
+            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows()
+                    .get(0)
+                    .size() == tableNonHeaderRow.size()) {
+                previousTableNonHeaderRow = previousTable.getRows().get(0);
+            }
+            if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
+                for (int i = currentTable.getRows()
+                        .size() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
+                    List<Cell> row = currentTable.getRows().get(i);
+                    if (row.size() == tableNonHeaderRow.size() && row.stream()
+                            .allMatch(cell -> cell.getHeaderCells().isEmpty())) {
+                        for (int j = 0; j < row.size(); j++) {
+                            row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
     private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline, Table previousTable) {
 
         Paragraph paragraph = new Paragraph();
@@ -73,6 +105,7 @@ public class SectionsBuilderService {
         Iterator<AbstractTextContainer> itty = wordBlockList.iterator();
         boolean alreadyAdded = false;
         AbstractTextContainer previous = null;
+        Table sectionTable = previousTable;
         while (itty.hasNext()) {
             AbstractTextContainer container = itty.next();
 
@@ -85,22 +118,8 @@ public class SectionsBuilderService {
                 } else {
                     table.setHeadline("Table in: " + lastHeadline);
                 }
-                // Distribute header information for subsequent tables
-                if (previousTable != null && hasInvalidHeaderInformation(table) && hasValidHeaderInformation(previousTable)) {
-                    List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
-                    List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(table);
-                    if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
-                        for (int i = table.getRows().size() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-                            List<Cell> row = table.getRows().get(i);
-                            if (row.size() == previousTableNonHeaderRow.size()
-                                    && row.stream().allMatch(cell -> cell.getHeaderCells().isEmpty())) {
-                                for (int j = 0; j < row.size(); j++) {
-                                    row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
-                                }
-                            }
-                        }
-                    }
-                }
+                mergeTableMetadata(table, sectionTable);
+                sectionTable = table;
 
                 if (textBlock != null && !alreadyAdded) {
                     paragraph.getPageBlocks().add(textBlock);

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -8,12 +8,6 @@ spring:
   profiles:
     active: kubernetes
 
-platform.multi-tenancy:
-  enabled: ${multitenancy.enabled:false}
-  tenantFilter:
-    urlPatterns: /redact
-    urlPatternsToIgnore:
-
 management:
   endpoint:
     metrics.enabled: ${monitoring.enabled:false}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java

@@ -232,7 +232,7 @@ public class EntityRedactionServiceTest {
                 "    then\n" +
                 "        section.redact(\"name\", 9, \"Redacted because row is a vertebrate study\");\n" +
                 "        section.redact(\"address\", 9, \"Redacted because rows is a vertebrate study\");\n" +
-                "        section.highlightCell(\"Vertebrate study Y/N\", 9);\n" +
+                "        section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" +
                 "    end";
         when(rulesClient.getVersion()).thenReturn(1L);
         when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -18,6 +18,7 @@ import org.springframework.test.context.junit4.SpringRunner;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
@@ -64,4 +65,84 @@ public class PdfSegmentationServiceTest {
         }
     }
 
+
+    @Test
+    public void testTableExtraction() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
+
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document document = pdfSegmentationService.parseDocument(pdDocument);
+            assertThat(document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())).isNotEmpty();
+            Table firstTable = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(0);
+            assertThat(firstTable.getColCount()).isEqualTo(8);
+            assertThat(firstTable.getRowCount()).isEqualTo(1);
+            Table secondTable = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(1);
+            assertThat(secondTable.getColCount()).isEqualTo(8);
+            assertThat(secondTable.getRowCount()).isEqualTo(2);
+            List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
+                    .get(0)
+                    .stream()
+                    .map(Cell::getHeaderCells)
+                    .collect(Collectors.toList());
+            assertThat(secondTable.getRows().stream()
+                    .allMatch(row -> row.stream()
+                            .map(Cell::getHeaderCells)
+                            .collect(Collectors.toList())
+                            .equals(firstTableHeaderCells)))
+                    .isTrue();
+        }
+    }
+
+
+    @Test
+    public void testMultiPageMetadataPropagation() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
+
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document document = pdfSegmentationService.parseDocument(pdDocument);
+            assertThat(document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())).isNotEmpty();
+            Table firstTable = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(0);
+            assertThat(firstTable.getColCount()).isEqualTo(9);
+            assertThat(firstTable.getRowCount()).isEqualTo(5);
+            Table secondTable = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(1);
+            assertThat(secondTable.getColCount()).isEqualTo(9);
+            assertThat(secondTable.getRowCount()).isEqualTo(6);
+            List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
+                    .get(firstTable.getRowCount() - 1)
+                    .stream()
+                    .map(Cell::getHeaderCells)
+                    .collect(Collectors.toList());
+            assertThat(secondTable.getRows().stream()
+                    .allMatch(row -> row.stream()
+                            .map(Cell::getHeaderCells)
+                            .collect(Collectors.toList())
+                            .equals(firstTableHeaderCells)))
+                    .isTrue();
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -90,28 +90,32 @@ rule "7: Redact contact information, if Producer is found"
 		section.redactBetween("No:", "Fax", "address", 7, "Producer was found");
 	end
 
-rule "8: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study"
-    when
-        Section(isVertebrateStudy())
-    then
-        section.redact("name", 8, "Redacted because row is a vertebrate study");
-        section.redact("address", 8, "Redacted because row is a vertebrate study");
-        section.highlightCell("Vertebrate study Y/N", 9);
-    end
 
-rule "9: Not redacted because Vertebrate Study = N"
+rule "8: Not redacted because Vertebrate Study = N"
     when
         Section(isNotVertebrateStudy())
     then
-        section.redactNot("name", 9, "Not redacted because row is not a vertebrate study");
-        section.redactNot("address", 9, "Not redacted because row is not a vertebrate study");
+        section.redactNot("name", 8, "Not redacted because row is not a vertebrate study");
+        section.redactNot("address", 8, "Not redacted because row is not a vertebrate study");
+        section.highlightCell("Vertebrate study Y/N", 8, "hint_only");
+        section.highlightCell("Verte brate study Y/N", 8, "hint_only");
     end
 
 
-rule "10: Redact if must redact entry is found"
+rule "9: Redact if must redact entry is found"
 	when
 		eval(section.contains("must_redact")==true);
 	then
-		section.redact("name", 10, "must_redact entry was found.");
-		section.redact("address", 10, "must_redact entry was found.");
-	end
+		section.redact("name", 9, "must_redact entry was found.");
+		section.redact("address", 9, "must_redact entry was found.");
+	end
+
+rule "10: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study"
+    when
+        Section(isVertebrateStudy())
+    then
+        section.redact("name", 10, "Redacted because row is a vertebrate study");
+        section.redact("address", 10, "Redacted because row is a vertebrate study");
+        section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
+        section.highlightCell("Verte brate study Y/N", 10, "must_redact");
+    end