Merge branch 'RED-7714' into 'main'

RED-7715 - Add log4j config to enable switching between json/line logs

See merge request fforesight/layout-parser!87

.gitlab-ci.yml

@@ -10,7 +10,7 @@ deploy:
   script:
     - echo "Building with gradle version ${BUILDVERSION}"
     - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
     - echo "BUILDVERSION=$BUILDVERSION" >> version.env
   artifacts:
     reports:

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -244,9 +244,9 @@ public class LayoutParsingPipeline {
 
     private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
 
-        //        if (!classificationPage.isLandscape()) {
+       if (!classificationPage.isLandscape()) {
         document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-//        }
+      }
         document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
         document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
         document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/CvTableParsingAdapter.java

@@ -8,6 +8,7 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.PageInfo;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 
@@ -24,24 +25,26 @@ public class CvTableParsingAdapter {
         Map<Integer, List<TableCells>> tableCells = new HashMap<>();
         tableServiceResponse.getData()
                 .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells())));
+                        .addAll(convertTableCells(tableData.getTableCells(), tableData.getPageInfo())));
 
         return tableCells;
     }
 
 
-    private Collection<TableCells> convertTableCells(List<TableCells> tableCells) {
+    private Collection<TableCells> convertTableCells(List<TableCells> tableCells, PageInfo pageInfo) {
 
         List<TableCells> cvParsedTableCells = new ArrayList<>();
 
-        tableCells.forEach(t -> cvParsedTableCells.add(TableCells.builder()
-                .y0(t.getY0())
-                .x1(t.getX1())
-                .y1(t.getY1())
-                .x0(t.getX0())
-                .width(t.getWidth())
-                .height(t.getHeight())
-                .build()));
+        tableCells.stream()
+                .filter(cell -> cell.getWidth() < pageInfo.getWidth() * 0.98 && cell.getHeight() < pageInfo.getHeight() * 0.98)
+                .forEach(t -> cvParsedTableCells.add(TableCells.builder()
+                        .y0(t.getY0())
+                        .x1(t.getX1())
+                        .y1(t.getY1())
+                        .x0(t.getX0())
+                        .width(t.getWidth())
+                        .height(t.getHeight())
+                        .build()));
 
         return cvParsedTableCells;
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/PDFLinesTextStripper.java

@@ -329,11 +329,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                 .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
     }
 
-    // !(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-    //                        .getUnicode()
-    //                        .equals("\t")))
-
-
     @Override
     public String getText(PDDocument doc) throws IOException {
 

layoutparser-service/layoutparser-service-server/src/main/resources/log4j2.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="INFO">
+    <Properties>
+        <property name="logType">${env:LOGGING_TYPE}</property>
+    </Properties>
+    <Appenders>
+        <Console name="json" target="SYSTEM_OUT">
+            <JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true"
+                        includeTimeMillis="true">
+
+                <KeyValuePair key="version" value="${project.version}"/>
+            </JSONLayout>
+        </Console>
+        <Console name="console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n " />
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="info">
+            <AppenderRef ref="${logType}"/>
+        </Root>
+    </Loggers>
+</Configuration>

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -61,12 +61,13 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     private SectionsBuilderService sectionsBuilderService;
 
 
-    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+    @SneakyThrows
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument, TableServiceResponse tableServiceResponse) {
 
         ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                 originDocument,
                 new ImageServiceResponse(),
-                new TableServiceResponse());
+                tableServiceResponse);
 
         redactManagerClassificationService.classifyDocument(classificationDocument);
 
@@ -76,6 +77,13 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     }
 
 
+    @SneakyThrows
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+
+        return buildClassificationDocument(originDocument, new TableServiceResponse());
+    }
+
+
     @Test
     public void tablesToHtmlDebugger() throws IOException {
 
@@ -88,6 +96,39 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     }
 
 
+    @Test
+    public void tablesToHtmlDebuggerWithCVResponse() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
+        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
+        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()), tableServiceResponse);
+
+        toHtml(document, "/tmp/ScanRotationBorder.html");
+    }
+
+
+    @Test
+    public void testScanRotationBorderIsIgnored() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
+        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
+        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()), tableServiceResponse);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+
+        // Quality of the table parsing is not good, because the file is rotated at scanning.
+        // We only asset that the table border is not the page border.
+        tables.forEach(table -> {
+            assertThat(table.getMinX()).isGreaterThan(75);
+            assertThat(table.getMaxX()).isLessThan(512);
+        });
+    }
+
+
     @Test
     @SneakyThrows
     public void testMapping() {