RED-6093: Prototype find entities in rules

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
 public class Document {
 
     private List<Page> pages = new ArrayList<>();
-    private List<Paragraph> paragraphs = new ArrayList<>();
+    private List<Section> sections = new ArrayList<>();
     private List<Header> headers = new ArrayList<>();
     private List<Footer> footers = new ArrayList<>();
     private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Section.java

@@ -1,7 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
@@ -10,10 +16,11 @@ import lombok.NoArgsConstructor;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 @Data
 @NoArgsConstructor
-public class Paragraph implements Comparable {
+public class Section implements Comparable {
 
     private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
     private List<PdfImage> images = new ArrayList<>();
@@ -62,4 +69,9 @@ public class Paragraph implements Comparable {
         return 0;
     }
 
+
+
+
+
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -185,6 +185,11 @@ public class SearchableText {
     }
 
 
+//    public List<String> getParagraphStrings(){
+//
+//    }
+
+
     public static String buildString(List<TextPositionSequence> sequences) {
 
         StringBuilder sb = new StringBuilder();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -21,6 +21,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 
@@ -34,8 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class ManualRedactionSurroundingTextService {
 
     private final RedactionStorageService redactionStorageService;
-    private final SurroundingWordsService surroundingWordsService;
+    private final RedactionServiceSettings redactionServiceSettings;
-
 
     @Timed("redactmanager_surroundingTextAnalysis")
     public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
@@ -87,9 +87,9 @@ public class ManualRedactionSurroundingTextService {
         }
 
         if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
+            SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
         } else {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
+            SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
         }
 
         return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -4,7 +4,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -31,7 +31,7 @@ public class SectionGridCreatorService {
 
     private void addSectionGrid(Document classifiedDoc, int page) {
 
-        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+        for (Section paragraph : classifiedDoc.getSections()) {
 
             for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -16,7 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
@@ -40,14 +40,14 @@ public class SectionTextBuilderService {
 
         List<SectionText> sectionTexts = new ArrayList<>();
         AtomicInteger sectionNumber = new AtomicInteger(1);
-        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+        for (Section section : classifiedDoc.getSections()) {
 
-            List<Table> tables = paragraph.getTables();
+            List<Table> tables = section.getTables();
             for (Table table : tables) {
                 sectionTexts.addAll(processTablePerRow(table, sectionNumber));
                 sectionNumber.incrementAndGet();
             }
-            sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph.getImages()));
+            sectionTexts.add(processText(section.getSearchableText(), section.getTextBlocks(), section.getHeadline(), sectionNumber, section.getImages()));
             sectionNumber.incrementAndGet();
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java

@@ -7,6 +7,7 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
 import org.springframework.stereotype.Service;
@@ -15,15 +16,12 @@ import java.util.List;
 import java.util.Set;
 
 @Slf4j
-@Service
+@UtilityClass
-@RequiredArgsConstructor
 public class SurroundingWordsService {
 
-    private final RedactionServiceSettings redactionServiceSettings;
-
 
     @Timed("redactmanager_addSurroundingText")
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
         if (entities.isEmpty()) {
             return;
@@ -35,7 +33,7 @@ public class SurroundingWordsService {
                 if (dictionary != null && dictionary.isHint(entity.getType())) {
                     continue;
                 }
-                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd());
+                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd(), surroundingWordsOffsetWindow, numberOfSurroundingWords);
             }
         } catch (Exception e) {
             log.warn("Could not get surrounding text!");
@@ -44,7 +42,7 @@ public class SurroundingWordsService {
 
 
     @Timed("redactmanager_addSurroundingTextTables")
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts) {
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
         if (entities.isEmpty()) {
             return;
@@ -75,7 +73,7 @@ public class SurroundingWordsService {
                         if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
                             int entityStartOffset = entity.getStart() - startOffset;
                             int entityEndOffset = entity.getEnd() - startOffset;
-                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
+                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset, surroundingWordsOffsetWindow, numberOfSurroundingWords);
                         }
                     }
                 }
@@ -86,23 +84,23 @@ public class SurroundingWordsService {
     }
 
 
-    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
+    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
-        int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        int offsetBefore = entityStartOffset - surroundingWordsOffsetWindow < 0 ? 0 : entityStartOffset - surroundingWordsOffsetWindow;
         String textBefore = text.substring(offsetBefore, entityStartOffset);
         if (!textBefore.isBlank()) {
             String[] wordsBefore = textBefore.split(" ");
-            int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsBefore.length;
+            int numberOfWordsBefore = wordsBefore.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsBefore.length;
             if (wordsBefore.length > 0) {
                 entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
             }
         }
 
-        int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        int endOffset = entityEndOffset + surroundingWordsOffsetWindow > text.length() ? text.length() : entityEndOffset + surroundingWordsOffsetWindow;
         String textAfter = text.substring(entityEndOffset, endOffset);
         if (!textAfter.isBlank()) {
             String[] wordsAfter = textAfter.split(" ");
-            int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsAfter.length;
+            int numberOfWordsAfter = wordsAfter.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsAfter.length;
             if (wordsAfter.length > 0) {
                 entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -15,7 +15,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@@ -32,7 +32,7 @@ public class SectionsBuilderService {
     public void buildSections(Document document) {
 
         List<AbstractTextContainer> chunkWords = new ArrayList<>();
-        List<Paragraph> chunkBlockList = new ArrayList<>();
+        List<Section> chunkBlockList = new ArrayList<>();
         List<Header> headers = new ArrayList<>();
         List<Footer> footers = new ArrayList<>();
         List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
@@ -69,7 +69,7 @@ public class SectionsBuilderService {
                 }
 
                 if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
-                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
+                    Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {
                         lastHeadline = current.getText();
@@ -101,11 +101,11 @@ public class SectionsBuilderService {
             }
         }
 
-        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
+        Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
         chunkBlock.setHeadline(lastHeadline);
         chunkBlockList.add(chunkBlock);
 
-        document.setParagraphs(chunkBlockList);
+        document.setSections(chunkBlockList);
         document.setHeaders(headers);
         document.setFooters(footers);
         document.setUnclassifiedTexts(unclassifiedTexts);
@@ -114,8 +114,8 @@ public class SectionsBuilderService {
 
     public void addImagesToSections(Document document) {
 
-        Map<Integer, List<Paragraph>> paragraphMap = new HashMap<>();
+        Map<Integer, List<Section>> paragraphMap = new HashMap<>();
-        for (Paragraph paragraph : document.getParagraphs()) {
+        for (Section paragraph : document.getSections()) {
             for (AbstractTextContainer container : paragraph.getPageBlocks()) {
 
                 paragraphMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()).add(paragraph);
@@ -124,22 +124,22 @@ public class SectionsBuilderService {
         }
 
         if (paragraphMap.isEmpty()) {
-            Paragraph paragraph = new Paragraph();
+            Section paragraph = new Section();
-            document.getParagraphs().add(paragraph);
+            document.getSections().add(paragraph);
             paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         // first page is always a paragraph, else we can't process pages 1..N,
         // where N is the first found page with a paragraph
         if (paragraphMap.get(1) == null) {
-            Paragraph paragraph = new Paragraph();
+            Section paragraph = new Section();
-            document.getParagraphs().add(paragraph);
+            document.getSections().add(paragraph);
             paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         for (Page page : document.getPages()) {
             for (PdfImage image : page.getImages()) {
-                List<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
+                List<Section> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
                 if (paragraphsOnPage == null) {
                     int i = page.getPageNumber();
                     while (paragraphsOnPage == null) {
@@ -147,7 +147,7 @@ public class SectionsBuilderService {
                         i--;
                     }
                 }
-                for (Paragraph paragraph : paragraphsOnPage) {
+                for (Section paragraph : paragraphsOnPage) {
                     Float xMin = null;
                     Float yMin = null;
                     Float xMax = null;
@@ -239,9 +239,9 @@ public class SectionsBuilderService {
     }
 
 
-    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
+    private Section buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
 
-        Paragraph paragraph = new Paragraph();
+        Section section = new Section();
         TextBlock textBlock = null;
 
         int pageBefore = -1;
@@ -268,40 +268,17 @@ public class SectionsBuilderService {
                 }
 
                 if (textBlock != null && !alreadyAdded) {
-                    paragraph.getPageBlocks().add(textBlock);
+                    section.getPageBlocks().add(textBlock);
                     alreadyAdded = true;
                 }
-                paragraph.getPageBlocks().add(table);
+                section.getPageBlocks().add(table);
                 continue;
             }
 
             TextBlock wordBlock = (TextBlock) container;
-
+            section.getPageBlocks().add(wordBlock);
-            if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-            } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-                alreadyAdded = false;
-            } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
-                textBlock.setPage(pageBefore);
-                paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-            } else {
-                TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
-            }
-            pageBefore = wordBlock.getPage();
-            splitByTable = false;
-            previous = container;
         }
-
+        return section;
-        if (textBlock != null && !alreadyAdded) {
-            paragraph.getPageBlocks().add(textBlock);
-        }
-        return paragraph;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -12,7 +12,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -37,7 +37,7 @@ public class PdfVisualisationService {
             PDPage pdPage = document.getPage(page - 1);
             PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
 
-            for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+            for (Section paragraph : classifiedDoc.getSections()) {
 
                 for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -114,6 +114,7 @@ public class PdfSegmentationServiceTest {
                 new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
     }
 
+
     @Test
     @SneakyThrows
     public void testMapping() {
@@ -144,8 +145,8 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
+        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(table.getColCount()).isEqualTo(6);
         assertThat(table.getRowCount()).isEqualTo(13);
         assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@@ -159,11 +160,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(2);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@@ -178,11 +179,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(9);
         assertThat(firstTable.getRowCount()).isEqualTo(5);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(9);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
@@ -197,11 +198,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@@ -239,38 +240,30 @@ public class PdfSegmentationServiceTest {
 
         validateTable(document, 0, 8, 9, 0, 2);
 
-        List<List<String>> values = Arrays.asList(
+        List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
-                Arrays.asList(
-                "Annex point Reference within DAR/RAR",
                         "Author, date",
                         "Study title",
                         "Analytical method Author, date, No.",
                         "Technique, LOQ of the method, validated working range",
                         "Method meets analytical validation criteria",
                         "Remarks (in case validation criteria are not met)",
-                        "Acceptability of the method"
+                        "Acceptability of the method"),
-                ),
+                Arrays.asList("",
-                Arrays.asList(
-                "",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                         "",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
-                ),
+                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
-                Arrays.asList(
-                        "CA 7.1.2.1.1 DAR (2009)",
                         "Evans P.G. 2001 TMJ4569B, VV-323245",
                         "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
                         "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
                         "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
                         "Y",
                         "N/A",
-                        "Y"
+                        "Y"));
-                )
-        );
 
         validateTable(document, 0, values);
 
@@ -503,6 +496,7 @@ public class PdfSegmentationServiceTest {
 
     }
 
+
     @Test
     public void testDoc15Page18() throws IOException {
 
@@ -517,11 +511,13 @@ public class PdfSegmentationServiceTest {
 
     }
 
+
     @Test
     public void testDoc28Page23() throws IOException {
 
         prepareStorage();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource(
+                "files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
 
@@ -530,7 +526,6 @@ public class PdfSegmentationServiceTest {
         validateTable(document, 0, 6, 8, 0, 2);
         validateTable(document, 1, 6, 8, 0, 1);
 
-
     }
 
 
@@ -548,6 +543,7 @@ public class PdfSegmentationServiceTest {
 
     }
 
+
     @Test
     public void testDoc30Page5() throws IOException {
 
@@ -563,10 +559,9 @@ public class PdfSegmentationServiceTest {
     }
 
 
-
     private void validateTable(Document document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {
 
-        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
         List<List<Cell>> rows = table.getRows();
         int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
 
@@ -576,9 +571,11 @@ public class PdfSegmentationServiceTest {
         assertThat(table.getRowCount()).isEqualTo(rowCount);
 
     }
+
+
     private void validateTable(Document document, int tableIndex, List<List<String>> values) {
 
-        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
         List<List<Cell>> rows = table.getRows();
 
         List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();
@@ -592,12 +589,11 @@ public class PdfSegmentationServiceTest {
 
     }
 
+
     private void validateTableSize(Document document, int tableSize) {
 
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().size()).isEqualTo(tableSize);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().size()).isEqualTo(tableSize);
 
     }
 
-
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -14,6 +14,12 @@ global Section section
 //        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
 //    end
 
+rule "-1: Find dictionary entries"
+    when
+    Section(findDictionaryEntities());
+    then
+    end
+
 
 rule "0: Add CBI_author from ai"
     when