diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java index 35381230..6ec6f70e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java @@ -1,15 +1,19 @@ package com.iqser.red.service.redaction.v1.server.classification.service; import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; import java.util.regex.Pattern; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.server.classification.model.Document; +import com.iqser.red.service.redaction.v1.server.classification.model.Orientation; import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils; +import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import lombok.RequiredArgsConstructor; @@ -52,11 +56,27 @@ public class ClassificationService { var bodyTextFrame = page.getBodyTextFrame(); + var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{1,2}\\s[A-Za-z]{3,50}", true); + var pattern2 = Patterns.getCompiledPattern(".*\\d$", true); + + Matcher matcher = pattern.matcher(textBlock.toString()); + Matcher matcher2 = pattern2.matcher(textBlock.toString()); + if (document.getFontSizeCounter().getMostPopular() == null) { textBlock.setClassification("Other"); return; } - if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter() + if (textBlock.getText().length() > 5 && textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 5.9 + + + && ((textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0))) + || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) + || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE")) && !textBlock.toString().endsWith(":")) { + textBlock.setClassification("H 1"); + document.setHeadlines(true); + + } + else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter() .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) { textBlock.setClassification("Header"); @@ -69,32 +89,39 @@ public class ClassificationService { if (!Pattern.matches("[0-9]+", textBlock.toString())) { textBlock.setClassification("Title"); } - } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter() - .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter() - .getCountPerValue() - .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences() - .get(0) - .getTextPositions() - .get(0) - .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { - - for (int i = 1; i <= headlineFontSizes.size(); i++) { - if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) { - textBlock.setClassification("H " + i); - document.setHeadlines(true); - } - } - } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, - textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter() - .getMostPopular() - .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences() - .get(0) - .getTextPositions() - .get(0) - .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { - textBlock.setClassification("H " + (headlineFontSizes.size() + 1)); + } + else if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) { + textBlock.setClassification("H 2"); document.setHeadlines(true); - } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() + } +// else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter() +// .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter() +// .getCountPerValue() +// .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences() +// .get(0) +// .getTextPositions() +// .get(0) +// .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { +// +// for (int i = 1; i <= headlineFontSizes.size(); i++) { +// if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) { +// textBlock.setClassification("H " + i); +// document.setHeadlines(true); +// } +// } +// } +// else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, +// textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter() +// .getMostPopular() +// .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences() +// .get(0) +// .getTextPositions() +// .get(0) +// .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { +// textBlock.setClassification("H " + (headlineFontSizes.size() + 1)); +// document.setHeadlines(true); +// } + else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) { textBlock.setClassification("TextBlock Bold"); } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java index fddcb26f..8e4c4db2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java @@ -68,7 +68,7 @@ public class SectionsBuilderService { continue; } - if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) { + if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) { Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline); chunkBlock.setHeadline(lastHeadline); if (document.isHeadlines()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java index fe006822..66756dac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java @@ -4,6 +4,7 @@ import static org.mockito.Mockito.when; import java.io.BufferedReader; import java.io.ByteArrayInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -51,9 +52,14 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; +import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.ChangeType; import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionRequest; +import com.iqser.red.service.redaction.v1.model.RedactionResult; import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; @@ -61,6 +67,7 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; +import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.StorageAutoConfiguration; @@ -133,6 +140,37 @@ public class HeadlinesGoldStandardIntegrationTest { private final static String TEST_FILE_ID = "123"; + @Test + public void extractHeadlines() throws IOException { + + AnalyzeRequest request = prepareStorage("files/RSS/31 - CA6375 - Acute Oral Toxicity - Rats.pdf"); + + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + analyzeService.analyze(request); + + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); + + String outputFileName = OsUtils.getTemporaryDirectory() + "/Headlines.pdf"; + + try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { + fileOutputStream.write(annotateResponse.getDocument()); + } + + + RedactionRequest redactionRequest = RedactionRequest.builder() + .dossierId(request.getDossierId()) + .fileId(request.getFileId()) + .dossierTemplateId(request.getDossierTemplateId()) + .build(); + + RedactionResult result1 = redactionController.classify(redactionRequest); + + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) { + fileOutputStream.write(result1.getDocument()); + } + } + + @Test public void testHeadlineDetection() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 38360d8b..e5cfe646 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -364,7 +364,7 @@ public class RedactionIntegrationTest { @Test public void titleExtraction() throws IOException { - AnalyzeRequest request = prepareStorage("files/RSS/06 - Isopyrazam - Acute Oral Toxicity Rat.pdf"); + AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -1111,7 +1111,7 @@ public class RedactionIntegrationTest { System.out.println("classificationTest"); - AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf"); + AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf"); RedactionRequest redactionRequest = RedactionRequest.builder() .dossierId(request.getDossierId())