Pull request #29: Fix multi-page table merge

Merge in RED/redaction-service from bugfix/multi-page-table-merge to master

* commit 'baa703928f50e53a66f925fcf72082df188482a1':
  Fix multi-page table merge

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.0.1</version>
+        <version>1.0.2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -32,21 +32,21 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.0.0</version>
+                <version>1.1.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>
 
-                <dependency>
-                    <groupId>org.apache.pdfbox</groupId>
-                    <artifactId>pdfbox</artifactId>
-                    <version>${pdfbox.version}</version>
-                </dependency>
-                <dependency>
-                    <groupId>org.apache.pdfbox</groupId>
-                    <artifactId>pdfbox-tools</artifactId>
-                    <version>${pdfbox.version}</version>
-                </dependency>
+            <dependency>
+                <groupId>org.apache.pdfbox</groupId>
+                <artifactId>pdfbox</artifactId>
+                <version>${pdfbox.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.pdfbox</groupId>
+                <artifactId>pdfbox-tools</artifactId>
+                <version>${pdfbox.version}</version>
+            </dependency>
 
         </dependencies>
 

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -3,11 +3,15 @@ package com.iqser.red.service.redaction.v1.model;
 import java.util.ArrayList;
 import java.util.List;
 
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class RedactionLogEntry {
 
     private String id;

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -63,15 +63,15 @@
         </dependency>
         <!-- commons -->
         <dependency>
-            <groupId>com.iqser.gin4.commons</groupId>
+            <groupId>com.iqser.red.commons</groupId>
             <artifactId>spring-commons</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.iqser.gin4.commons</groupId>
+            <groupId>com.iqser.red.commons</groupId>
             <artifactId>logging-commons</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.iqser.gin4.commons</groupId>
+            <groupId>com.iqser.red.commons</groupId>
             <artifactId>metric-commons</artifactId>
         </dependency>
         <!-- other external -->
@@ -104,7 +104,7 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.iqser.gin4.commons</groupId>
+            <groupId>com.iqser.red.commons</groupId>
             <artifactId>test-commons</artifactId>
             <scope>test</scope>
         </dependency>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -20,7 +20,7 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
-import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/ControllerAdvice.java

@@ -2,13 +2,13 @@ package com.iqser.red.service.redaction.v1.server.controller;
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.commons.spring.ErrorMessage;
 import org.springframework.http.HttpStatus;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestControllerAdvice;
 
-import com.iqser.gin4.commons.api.errorhandling.ErrorMessage;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 
 import lombok.extern.slf4j.Slf4j;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -42,13 +42,17 @@ public class SectionsBuilderService {
                     Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline, previousTable);
                     chunkBlock.setHeadline(lastHeadline);
                     lastHeadline = current.getText();
-                    if (CollectionUtils.isNotEmpty(chunkBlock.getTables())) {
-                        previousTable = chunkBlock.getTables().get(0);
-                    }
                     chunkBlockList.add(chunkBlock);
                     chunkWords = new ArrayList<>();
+                    if (CollectionUtils.isNotEmpty(chunkBlock.getTables())) {
+                        previousTable = chunkBlock.getTables().get(chunkBlock.getTables().size() - 1);
+                    }
+                } else if (current instanceof Table) {
+                    Table table = (Table) current;
+                    // Distribute header information for subsequent tables
+                    mergeTableMetadata(table, previousTable);
+                    previousTable = table;
                 }
-
                 chunkWords.add(current);
                 prev = current;
             }
@@ -62,6 +66,34 @@ public class SectionsBuilderService {
     }
 
 
+    private void mergeTableMetadata(Table currentTable, Table previousTable) {
+
+        // Distribute header information for subsequent tables
+        if (previousTable != null && hasInvalidHeaderInformation(currentTable) && hasValidHeaderInformation(previousTable)) {
+            List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
+            List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
+            // Allow merging of tables if header row is separated from first logical non-header row
+            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows()
+                    .get(0)
+                    .size() == tableNonHeaderRow.size()) {
+                previousTableNonHeaderRow = previousTable.getRows().get(0);
+            }
+            if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
+                for (int i = currentTable.getRows()
+                        .size() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
+                    List<Cell> row = currentTable.getRows().get(i);
+                    if (row.size() == tableNonHeaderRow.size() && row.stream()
+                            .allMatch(cell -> cell.getHeaderCells().isEmpty())) {
+                        for (int j = 0; j < row.size(); j++) {
+                            row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
     private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline, Table previousTable) {
 
         Paragraph paragraph = new Paragraph();
@@ -73,6 +105,7 @@ public class SectionsBuilderService {
         Iterator<AbstractTextContainer> itty = wordBlockList.iterator();
         boolean alreadyAdded = false;
         AbstractTextContainer previous = null;
+        Table sectionTable = previousTable;
         while (itty.hasNext()) {
             AbstractTextContainer container = itty.next();
 
@@ -85,22 +118,8 @@ public class SectionsBuilderService {
                 } else {
                     table.setHeadline("Table in: " + lastHeadline);
                 }
-                // Distribute header information for subsequent tables
-                if (previousTable != null && hasInvalidHeaderInformation(table) && hasValidHeaderInformation(previousTable)) {
-                    List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
-                    List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(table);
-                    if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
-                        for (int i = table.getRows().size() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-                            List<Cell> row = table.getRows().get(i);
-                            if (row.size() == previousTableNonHeaderRow.size()
-                                    && row.stream().allMatch(cell -> cell.getHeaderCells().isEmpty())) {
-                                for (int j = 0; j < row.size(); j++) {
-                                    row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
-                                }
-                            }
-                        }
-                    }
-                }
+                mergeTableMetadata(table, sectionTable);
+                sectionTable = table;
 
                 if (textBlock != null && !alreadyAdded) {
                     paragraph.getPageBlocks().add(textBlock);

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -8,12 +8,6 @@ spring:
   profiles:
     active: kubernetes
 
-platform.multi-tenancy:
-  enabled: ${multitenancy.enabled:false}
-  tenantFilter:
-    urlPatterns: /redact
-    urlPatternsToIgnore:
-
 management:
   endpoint:
     metrics.enabled: ${monitoring.enabled:false}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -18,6 +18,7 @@ import org.springframework.test.context.junit4.SpringRunner;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
@@ -64,4 +65,84 @@ public class PdfSegmentationServiceTest {
         }
     }
 
+
+    @Test
+    public void testTableExtraction() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
+
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document document = pdfSegmentationService.parseDocument(pdDocument);
+            assertThat(document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())).isNotEmpty();
+            Table firstTable = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(0);
+            assertThat(firstTable.getColCount()).isEqualTo(8);
+            assertThat(firstTable.getRowCount()).isEqualTo(1);
+            Table secondTable = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(1);
+            assertThat(secondTable.getColCount()).isEqualTo(8);
+            assertThat(secondTable.getRowCount()).isEqualTo(2);
+            List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
+                    .get(0)
+                    .stream()
+                    .map(Cell::getHeaderCells)
+                    .collect(Collectors.toList());
+            assertThat(secondTable.getRows().stream()
+                    .allMatch(row -> row.stream()
+                            .map(Cell::getHeaderCells)
+                            .collect(Collectors.toList())
+                            .equals(firstTableHeaderCells)))
+                    .isTrue();
+        }
+    }
+
+
+    @Test
+    public void testMultiPageMetadataPropagation() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
+
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document document = pdfSegmentationService.parseDocument(pdDocument);
+            assertThat(document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())).isNotEmpty();
+            Table firstTable = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(0);
+            assertThat(firstTable.getColCount()).isEqualTo(9);
+            assertThat(firstTable.getRowCount()).isEqualTo(5);
+            Table secondTable = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(1);
+            assertThat(secondTable.getColCount()).isEqualTo(9);
+            assertThat(secondTable.getRowCount()).isEqualTo(6);
+            List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
+                    .get(firstTable.getRowCount() - 1)
+                    .stream()
+                    .map(Cell::getHeaderCells)
+                    .collect(Collectors.toList());
+            assertThat(secondTable.getRows().stream()
+                    .allMatch(row -> row.stream()
+                            .map(Cell::getHeaderCells)
+                            .collect(Collectors.toList())
+                            .equals(firstTableHeaderCells)))
+                    .isTrue();
+        }
+    }
+
 }