Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6093: Prototype find entities in rules

Browse Source
This commit is contained in:
deiflaender 2023-02-13 15:49:20 +01:00 committed by Kilian Schuettler
parent 2646407805
commit a1c5be1e3e
11 changed files with 78 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java
View File

@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
public class Document {
private List<Page> pages = new ArrayList<>();
private List<Paragraph> paragraphs = new ArrayList<>();
private List<Section> sections = new ArrayList<>();
private List<Header> headers = new ArrayList<>();
private List<Footer> footers = new ArrayList<>();
private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Paragraph.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Section.java
View File

@ -1,7 +1,13 @@
package com.iqser.red.service.redaction.v1.server.classification.model;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@ -10,10 +16,11 @@ import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@Data
@NoArgsConstructor
public class Paragraph implements Comparable {
public class Section implements Comparable {
private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
private List<PdfImage> images = new ArrayList<>();
@ -62,4 +69,9 @@ public class Paragraph implements Comparable {
return 0;
}
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java
View File

@ -185,6 +185,11 @@ public class SearchableText {
}
// public List<String> getParagraphStrings(){
//
// }
public static String buildString(List<TextPositionSequence> sequences) {
StringBuilder sb = new StringBuilder();

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java
View File

@ -21,6 +21,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
@ -34,8 +35,7 @@ import lombok.extern.slf4j.Slf4j;
public class ManualRedactionSurroundingTextService {
private final RedactionStorageService redactionStorageService;
private final SurroundingWordsService surroundingWordsService;
private final RedactionServiceSettings redactionServiceSettings;
@Timed("redactmanager_surroundingTextAnalysis")
public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
@ -87,9 +87,9 @@ public class ManualRedactionSurroundingTextService {
}
if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
} else {
surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
}
return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java
View File

@ -4,7 +4,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@ -31,7 +31,7 @@ public class SectionGridCreatorService {
private void addSectionGrid(Document classifiedDoc, int page) {
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
for (Section paragraph : classifiedDoc.getSections()) {
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java
View File

@ -16,7 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
@ -40,14 +40,14 @@ public class SectionTextBuilderService {
List<SectionText> sectionTexts = new ArrayList<>();
AtomicInteger sectionNumber = new AtomicInteger(1);
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
for (Section section : classifiedDoc.getSections()) {
List<Table> tables = paragraph.getTables();
List<Table> tables = section.getTables();
for (Table table : tables) {
sectionTexts.addAll(processTablePerRow(table, sectionNumber));
sectionNumber.incrementAndGet();
}
sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph.getImages()));
sectionTexts.add(processText(section.getSearchableText(), section.getTextBlocks(), section.getHeadline(), sectionNumber, section.getImages()));
sectionNumber.incrementAndGet();
}

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java
View File

@ -7,6 +7,7 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -15,15 +16,12 @@ import java.util.List;
import java.util.Set;
@Slf4j
@Service
@RequiredArgsConstructor
@UtilityClass
public class SurroundingWordsService {
private final RedactionServiceSettings redactionServiceSettings;
@Timed("redactmanager_addSurroundingText")
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
if (entities.isEmpty()) {
return;
@ -35,7 +33,7 @@ public class SurroundingWordsService {
if (dictionary != null && dictionary.isHint(entity.getType())) {
continue;
}
findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd());
findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd(), surroundingWordsOffsetWindow, numberOfSurroundingWords);
}
} catch (Exception e) {
log.warn("Could not get surrounding text!");
@ -44,7 +42,7 @@ public class SurroundingWordsService {
@Timed("redactmanager_addSurroundingTextTables")
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts) {
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
if (entities.isEmpty()) {
return;
@ -75,7 +73,7 @@ public class SurroundingWordsService {
if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
int entityStartOffset = entity.getStart() - startOffset;
int entityEndOffset = entity.getEnd() - startOffset;
findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
findSurroundingWords(entity, text, entityStartOffset, entityEndOffset, surroundingWordsOffsetWindow, numberOfSurroundingWords);
}
}
}
@ -86,23 +84,23 @@ public class SurroundingWordsService {
}
private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow();
int offsetBefore = entityStartOffset - surroundingWordsOffsetWindow < 0 ? 0 : entityStartOffset - surroundingWordsOffsetWindow;
String textBefore = text.substring(offsetBefore, entityStartOffset);
if (!textBefore.isBlank()) {
String[] wordsBefore = textBefore.split(" ");
int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsBefore.length;
int numberOfWordsBefore = wordsBefore.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsBefore.length;
if (wordsBefore.length > 0) {
entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
}
}
int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
int endOffset = entityEndOffset + surroundingWordsOffsetWindow > text.length() ? text.length() : entityEndOffset + surroundingWordsOffsetWindow;
String textAfter = text.substring(entityEndOffset, endOffset);
if (!textAfter.isBlank()) {
String[] wordsAfter = textAfter.split(" ");
int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsAfter.length;
int numberOfWordsAfter = wordsAfter.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsAfter.length;
if (wordsAfter.length > 0) {
entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
}

61
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java
View File

@ -15,7 +15,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@ -32,7 +32,7 @@ public class SectionsBuilderService {
public void buildSections(Document document) {
List<AbstractTextContainer> chunkWords = new ArrayList<>();
List<Paragraph> chunkBlockList = new ArrayList<>();
List<Section> chunkBlockList = new ArrayList<>();
List<Header> headers = new ArrayList<>();
List<Footer> footers = new ArrayList<>();
List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
@ -69,7 +69,7 @@ public class SectionsBuilderService {
}
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
chunkBlock.setHeadline(lastHeadline);
if (document.isHeadlines()) {
lastHeadline = current.getText();
@ -101,11 +101,11 @@ public class SectionsBuilderService {
}
}
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
chunkBlock.setHeadline(lastHeadline);
chunkBlockList.add(chunkBlock);
document.setParagraphs(chunkBlockList);
document.setSections(chunkBlockList);
document.setHeaders(headers);
document.setFooters(footers);
document.setUnclassifiedTexts(unclassifiedTexts);
@ -114,8 +114,8 @@ public class SectionsBuilderService {
public void addImagesToSections(Document document) {
Map<Integer, List<Paragraph>> paragraphMap = new HashMap<>();
for (Paragraph paragraph : document.getParagraphs()) {
Map<Integer, List<Section>> paragraphMap = new HashMap<>();
for (Section paragraph : document.getSections()) {
for (AbstractTextContainer container : paragraph.getPageBlocks()) {
paragraphMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()).add(paragraph);
@ -124,22 +124,22 @@ public class SectionsBuilderService {
}
if (paragraphMap.isEmpty()) {
Paragraph paragraph = new Paragraph();
document.getParagraphs().add(paragraph);
Section paragraph = new Section();
document.getSections().add(paragraph);
paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
}
// first page is always a paragraph, else we can't process pages 1..N,
// where N is the first found page with a paragraph
if (paragraphMap.get(1) == null) {
Paragraph paragraph = new Paragraph();
document.getParagraphs().add(paragraph);
Section paragraph = new Section();
document.getSections().add(paragraph);
paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
}
for (Page page : document.getPages()) {
for (PdfImage image : page.getImages()) {
List<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
List<Section> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
if (paragraphsOnPage == null) {
int i = page.getPageNumber();
while (paragraphsOnPage == null) {
@ -147,7 +147,7 @@ public class SectionsBuilderService {
i--;
}
}
for (Paragraph paragraph : paragraphsOnPage) {
for (Section paragraph : paragraphsOnPage) {
Float xMin = null;
Float yMin = null;
Float xMax = null;
@ -239,9 +239,9 @@ public class SectionsBuilderService {
}
private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
private Section buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
Paragraph paragraph = new Paragraph();
Section section = new Section();
TextBlock textBlock = null;
int pageBefore = -1;
@ -268,40 +268,17 @@ public class SectionsBuilderService {
}
if (textBlock != null && !alreadyAdded) {
paragraph.getPageBlocks().add(textBlock);
section.getPageBlocks().add(textBlock);
alreadyAdded = true;
}
paragraph.getPageBlocks().add(table);
section.getPageBlocks().add(table);
continue;
}
TextBlock wordBlock = (TextBlock) container;
if (textBlock == null) {
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
textBlock.setPage(wordBlock.getPage());
} else if (splitByTable) {
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
textBlock.setPage(wordBlock.getPage());
alreadyAdded = false;
} else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
textBlock.setPage(pageBefore);
paragraph.getPageBlocks().add(textBlock);
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
textBlock.setPage(wordBlock.getPage());
} else {
TextBlock spatialEntity = textBlock.union(wordBlock);
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
}
pageBefore = wordBlock.getPage();
splitByTable = false;
previous = container;
section.getPageBlocks().add(wordBlock);
}
if (textBlock != null && !alreadyAdded) {
paragraph.getPageBlocks().add(textBlock);
}
return paragraph;
return section;
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java
View File

@ -12,7 +12,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@ -37,7 +37,7 @@ public class PdfVisualisationService {
PDPage pdPage = document.getPage(page - 1);
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
for (Section paragraph : classifiedDoc.getSections()) {
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {

22
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java
View File

@ -135,8 +135,8 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(table.getColCount()).isEqualTo(6);
assertThat(table.getRowCount()).isEqualTo(13);
assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@ -150,11 +150,11 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(firstTable.getColCount()).isEqualTo(8);
assertThat(firstTable.getRowCount()).isEqualTo(1);
Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
assertThat(secondTable.getColCount()).isEqualTo(8);
assertThat(secondTable.getRowCount()).isEqualTo(2);
List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@ -169,11 +169,11 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(firstTable.getColCount()).isEqualTo(9);
assertThat(firstTable.getRowCount()).isEqualTo(5);
Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
assertThat(secondTable.getColCount()).isEqualTo(9);
assertThat(secondTable.getRowCount()).isEqualTo(6);
List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
@ -188,11 +188,11 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(firstTable.getColCount()).isEqualTo(8);
assertThat(firstTable.getRowCount()).isEqualTo(1);
Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
assertThat(secondTable.getColCount()).isEqualTo(8);
assertThat(secondTable.getRowCount()).isEqualTo(6);
List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -14,6 +14,12 @@ global Section section
// section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
// end
rule "-1: Find dictionary entries"
when
Section(findDictionaryEntities());
then
end
rule "0: Add CBI_author from ai"
when