RED-6093: Prototype find entities in rules

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
 public class Document {
 
     private List<Page> pages = new ArrayList<>();
-    private List<Paragraph> paragraphs = new ArrayList<>();
+    private List<Section> sections = new ArrayList<>();
     private List<Header> headers = new ArrayList<>();
     private List<Footer> footers = new ArrayList<>();
     private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Section.java

@@ -1,7 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
@@ -10,10 +16,11 @@ import lombok.NoArgsConstructor;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 @Data
 @NoArgsConstructor
-public class Paragraph implements Comparable {
+public class Section implements Comparable {
 
     private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
     private List<PdfImage> images = new ArrayList<>();
@@ -62,4 +69,9 @@ public class Paragraph implements Comparable {
         return 0;
     }
 
+
+
+
+
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -187,6 +187,11 @@ public class SearchableText {
     }
 
 
+//    public List<String> getParagraphStrings(){
+//
+//    }
+
+
     public static String buildString(List<TextPositionSequence> sequences) {
 
         StringBuilder sb = new StringBuilder();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -18,21 +18,28 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
+import io.micrometer.core.annotation.Timed;
 import lombok.Builder;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
@@ -85,6 +92,11 @@ public class Section {
 
     private boolean isInTable;
 
+    @Builder.Default
+    private List<Integer> cellStarts = new ArrayList<>();
+
+    private RedactionServiceSettings redactionServiceSettings;
+
 
     @Deprecated
     @SuppressWarnings("unused")
@@ -1675,6 +1687,98 @@ public class Section {
 
     }
 
+
+
+    public boolean findDictionaryEntities(){
+
+        findEntities();
+
+        if (cellStarts != null && !cellStarts.isEmpty()) {
+            SurroundingWordsService.addSurroundingText(entities, searchableText, dictionary, cellStarts, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
+        } else {
+            SurroundingWordsService.addSurroundingText(entities, searchableText, dictionary, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
+        }
+
+        if (!isLocal && manualRedactions != null) {
+
+            var approvedForceRedactions = manualRedactions
+                    .getForceRedactions()
+                    .stream()
+                    .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
+                    .filter(fr -> fr.getRequestDate() != null)
+                    .collect(Collectors.toList());
+            // only approved id removals, that haven't been forced back afterwards
+            var idsToRemove = manualRedactions
+                    .getIdsToRemove()
+                    .stream()
+                    .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
+                    .filter(idr -> idr.getRequestDate() != null)
+                    .filter(idr -> approvedForceRedactions.stream()
+                            .noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
+                                    .isAfter(idr.getRequestDate())))
+                    .map(IdRemoval::getAnnotationId)
+                    .collect(Collectors.toSet());
+
+            if (images != null && !images.isEmpty() && manualRedactions.getImageRecategorization() != null) {
+                for (Image image : images) {
+                    String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
+                    for (ManualImageRecategorization imageRecategorization : manualRedactions.getImageRecategorization()) {
+                        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
+                            image.setType(imageRecategorization.getType());
+                        }
+                    }
+                    if (idsToRemove.contains(imageId)) {
+                        image.setIgnored(true);
+                    }
+                }
+            }
+
+            entities.forEach(entity -> entity.getPositionSequences().forEach(ps -> {
+                if (idsToRemove.contains(ps.getId())) {
+                    entity.setIgnored(true);
+                }
+            }));
+        }
+
+        return false;
+    }
+
+
+    private void findEntities() {
+
+        Set<Entity> found = new HashSet<>();
+        String searchableString = searchableText.asString();
+
+        if (StringUtils.isEmpty(searchableString)) {
+            entities = new HashSet<>();
+            return;
+        }
+
+        String lowercaseInputString = searchableString.toLowerCase();
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+
+            var searchImplementation = isLocal ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation,
+                    model,
+                    new FindEntityDetails(model.getType(),
+                            headline,
+                            sectionNumber,
+                            !isLocal,
+                            model.isDossierDictionary(),
+                            isLocal ? Engine.RULE : Engine.DICTIONARY,
+                            isLocal ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+
+            EntitySearchUtils.addOrAddEngine(found, entities);
+        }
+
+        entities = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        nerEntities = EntitySearchUtils.clearAndFindPositions(nerEntities, searchableText, dictionary, manualRedactions);
+    }
+
+
+
+
 }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -35,7 +35,6 @@ public class EntityRedactionService {
 
     private final RedactionServiceSettings redactionServiceSettings;
     private final DroolsExecutionService droolsExecutionService;
-    private final SurroundingWordsService surroundingWordsService;
 
 
     public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@@ -78,60 +77,9 @@ public class EntityRedactionService {
         List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
         for (SectionText reanalysisSection : reanalysisSections) {
 
-            Entities entities = findEntities(reanalysisSection.getSearchableText(),
+            Set<Entity> nerFound = new HashSet<>();
-                    reanalysisSection.getHeadline(),
+            if (!local) {
-                    reanalysisSection.getSectionNumber(),
+                nerFound.addAll(getNerValues(reanalysisSection.getSectionNumber(), nerEntities, reanalysisSection.getCellStarts(), reanalysisSection.getHeadline()));
-                    dictionary,
-                    local,
-                    nerEntities,
-                    reanalysisSection.getCellStarts(),
-                    analyzeRequest.getManualRedactions());
-
-            if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
-            }
-
-            if (!local && analyzeRequest.getManualRedactions() != null) {
-
-                var approvedForceRedactions = analyzeRequest.getManualRedactions()
-                        .getForceRedactions()
-                        .stream()
-                        .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
-                        .filter(fr -> fr.getRequestDate() != null)
-                        .collect(Collectors.toList());
-                // only approved id removals, that haven't been forced back afterwards
-                var idsToRemove = analyzeRequest.getManualRedactions()
-                        .getIdsToRemove()
-                        .stream()
-                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
-                        .filter(idr -> idr.getRequestDate() != null)
-                        .filter(idr -> approvedForceRedactions.stream()
-                                .noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
-                                        .isAfter(idr.getRequestDate())))
-                        .map(IdRemoval::getAnnotationId)
-                        .collect(Collectors.toSet());
-
-                if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
-                    for (Image image : reanalysisSection.getImages()) {
-                        String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
-                        for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions().getImageRecategorization()) {
-                            if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
-                                image.setType(imageRecategorization.getType());
-                            }
-                        }
-                        if (idsToRemove.contains(imageId)) {
-                            image.setIgnored(true);
-                        }
-                    }
-                }
-
-                entities.getEntities().forEach(entity -> entity.getPositionSequences().forEach(ps -> {
-                    if (idsToRemove.contains(ps.getId())) {
-                        entity.setIgnored(true);
-                    }
-                }));
             }
 
             log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
@@ -139,9 +87,7 @@ public class EntityRedactionService {
             sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                     .isLocal(false)
                     .dictionaryTypes(dictionary.getTypes())
-                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
+                    .nerEntities(nerFound)
-                            .stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
-                    .nerEntities(entities.getNerEntities())
                     .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
                     .searchText(reanalysisSection.getSearchableText().toString())
                     .headline(reanalysisSection.getHeadline())
@@ -154,6 +100,8 @@ public class EntityRedactionService {
                     .fileAttributes(analyzeRequest.getFileAttributes())
                     .manualRedactions(analyzeRequest.getManualRedactions())
                     .isInTable(reanalysisSection.isTable())
+                    .redactionServiceSettings(redactionServiceSettings)
+                    .cellStarts(reanalysisSection.getCellStarts())
                     .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
 
         }
@@ -183,12 +131,12 @@ public class EntityRedactionService {
                     .collect(Collectors.toSet());
 
             if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
                         sectionSearchableTextPair.getSearchableText(),
                         dictionary,
-                        sectionSearchableTextPair.getCellStarts());
+                        sectionSearchableTextPair.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
             } else {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
+                SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
             }
 
             entities.addAll(analysedSection.getEntities());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -34,8 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class ManualRedactionSurroundingTextService {
 
     private final RedactionStorageService redactionStorageService;
-    private final SurroundingWordsService surroundingWordsService;
+    private final RedactionServiceSettings redactionServiceSettings;
-
 
     @Timed("redactmanager_surroundingTextAnalysis")
     public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
@@ -87,9 +87,9 @@ public class ManualRedactionSurroundingTextService {
         }
 
         if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
+            SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
         } else {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
+            SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
         }
 
         return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -4,7 +4,7 @@ import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -31,7 +31,7 @@ public class SectionGridCreatorService {
 
     private void addSectionGrid(Document classifiedDoc, int page) {
 
-        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+        for (Section paragraph : classifiedDoc.getSections()) {
 
             for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -16,7 +16,7 @@ import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
@@ -40,14 +40,14 @@ public class SectionTextBuilderService {
 
         List<SectionText> sectionTexts = new ArrayList<>();
         AtomicInteger sectionNumber = new AtomicInteger(1);
-        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+        for (Section section : classifiedDoc.getSections()) {
 
-            List<Table> tables = paragraph.getTables();
+            List<Table> tables = section.getTables();
             for (Table table : tables) {
                 sectionTexts.addAll(processTablePerRow(table, sectionNumber));
                 sectionNumber.incrementAndGet();
             }
-            sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph.getImages()));
+            sectionTexts.add(processText(section.getSearchableText(), section.getTextBlocks(), section.getHeadline(), sectionNumber, section.getImages()));
             sectionNumber.incrementAndGet();
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java

@@ -7,6 +7,7 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
 import org.springframework.stereotype.Service;
@@ -15,15 +16,12 @@ import java.util.List;
 import java.util.Set;
 
 @Slf4j
-@Service
+@UtilityClass
-@RequiredArgsConstructor
 public class SurroundingWordsService {
 
-    private final RedactionServiceSettings redactionServiceSettings;
-
 
     @Timed("redactmanager_addSurroundingText")
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
         if (entities.isEmpty()) {
             return;
@@ -35,7 +33,7 @@ public class SurroundingWordsService {
                 if (dictionary != null && dictionary.isHint(entity.getType())) {
                     continue;
                 }
-                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd());
+                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd(), surroundingWordsOffsetWindow, numberOfSurroundingWords);
             }
         } catch (Exception e) {
             log.warn("Could not get surrounding text!");
@@ -44,7 +42,7 @@ public class SurroundingWordsService {
 
 
     @Timed("redactmanager_addSurroundingTextTables")
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts) {
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
         if (entities.isEmpty()) {
             return;
@@ -75,7 +73,7 @@ public class SurroundingWordsService {
                         if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
                             int entityStartOffset = entity.getStart() - startOffset;
                             int entityEndOffset = entity.getEnd() - startOffset;
-                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
+                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset, surroundingWordsOffsetWindow, numberOfSurroundingWords);
                         }
                     }
                 }
@@ -86,23 +84,23 @@ public class SurroundingWordsService {
     }
 
 
-    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
+    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
-        int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        int offsetBefore = entityStartOffset - surroundingWordsOffsetWindow < 0 ? 0 : entityStartOffset - surroundingWordsOffsetWindow;
         String textBefore = text.substring(offsetBefore, entityStartOffset);
         if (!textBefore.isBlank()) {
             String[] wordsBefore = textBefore.split(" ");
-            int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsBefore.length;
+            int numberOfWordsBefore = wordsBefore.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsBefore.length;
             if (wordsBefore.length > 0) {
                 entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
             }
         }
 
-        int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        int endOffset = entityEndOffset + surroundingWordsOffsetWindow > text.length() ? text.length() : entityEndOffset + surroundingWordsOffsetWindow;
         String textAfter = text.substring(entityEndOffset, endOffset);
         if (!textAfter.isBlank()) {
             String[] wordsAfter = textAfter.split(" ");
-            int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsAfter.length;
+            int numberOfWordsAfter = wordsAfter.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsAfter.length;
             if (wordsAfter.length > 0) {
                 entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -15,7 +15,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@@ -32,7 +32,7 @@ public class SectionsBuilderService {
     public void buildSections(Document document) {
 
         List<AbstractTextContainer> chunkWords = new ArrayList<>();
-        List<Paragraph> chunkBlockList = new ArrayList<>();
+        List<Section> chunkBlockList = new ArrayList<>();
         List<Header> headers = new ArrayList<>();
         List<Footer> footers = new ArrayList<>();
         List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
@@ -69,7 +69,7 @@ public class SectionsBuilderService {
                 }
 
                 if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
-                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
+                    Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {
                         lastHeadline = current.getText();
@@ -101,11 +101,11 @@ public class SectionsBuilderService {
             }
         }
 
-        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
+        Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
         chunkBlock.setHeadline(lastHeadline);
         chunkBlockList.add(chunkBlock);
 
-        document.setParagraphs(chunkBlockList);
+        document.setSections(chunkBlockList);
         document.setHeaders(headers);
         document.setFooters(footers);
         document.setUnclassifiedTexts(unclassifiedTexts);
@@ -114,8 +114,8 @@ public class SectionsBuilderService {
 
     public void addImagesToSections(Document document) {
 
-        Map<Integer, List<Paragraph>> paragraphMap = new HashMap<>();
+        Map<Integer, List<Section>> paragraphMap = new HashMap<>();
-        for (Paragraph paragraph : document.getParagraphs()) {
+        for (Section paragraph : document.getSections()) {
             for (AbstractTextContainer container : paragraph.getPageBlocks()) {
 
                 paragraphMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()).add(paragraph);
@@ -124,22 +124,22 @@ public class SectionsBuilderService {
         }
 
         if (paragraphMap.isEmpty()) {
-            Paragraph paragraph = new Paragraph();
+            Section paragraph = new Section();
-            document.getParagraphs().add(paragraph);
+            document.getSections().add(paragraph);
             paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         // first page is always a paragraph, else we can't process pages 1..N,
         // where N is the first found page with a paragraph
         if (paragraphMap.get(1) == null) {
-            Paragraph paragraph = new Paragraph();
+            Section paragraph = new Section();
-            document.getParagraphs().add(paragraph);
+            document.getSections().add(paragraph);
             paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         for (Page page : document.getPages()) {
             for (PdfImage image : page.getImages()) {
-                List<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
+                List<Section> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
                 if (paragraphsOnPage == null) {
                     int i = page.getPageNumber();
                     while (paragraphsOnPage == null) {
@@ -147,7 +147,7 @@ public class SectionsBuilderService {
                         i--;
                     }
                 }
-                for (Paragraph paragraph : paragraphsOnPage) {
+                for (Section paragraph : paragraphsOnPage) {
                     Float xMin = null;
                     Float yMin = null;
                     Float xMax = null;
@@ -239,9 +239,9 @@ public class SectionsBuilderService {
     }
 
 
-    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
+    private Section buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
 
-        Paragraph paragraph = new Paragraph();
+        Section section = new Section();
         TextBlock textBlock = null;
 
         int pageBefore = -1;
@@ -268,40 +268,17 @@ public class SectionsBuilderService {
                 }
 
                 if (textBlock != null && !alreadyAdded) {
-                    paragraph.getPageBlocks().add(textBlock);
+                    section.getPageBlocks().add(textBlock);
                     alreadyAdded = true;
                 }
-                paragraph.getPageBlocks().add(table);
+                section.getPageBlocks().add(table);
                 continue;
             }
 
             TextBlock wordBlock = (TextBlock) container;
-
+            section.getPageBlocks().add(wordBlock);
-            if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-            } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-                alreadyAdded = false;
-            } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
-                textBlock.setPage(pageBefore);
-                paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-            } else {
-                TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
-            }
-            pageBefore = wordBlock.getPage();
-            splitByTable = false;
-            previous = container;
         }
-
+        return section;
-        if (textBlock != null && !alreadyAdded) {
-            paragraph.getPageBlocks().add(textBlock);
-        }
-        return paragraph;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -12,7 +12,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -37,7 +37,7 @@ public class PdfVisualisationService {
             PDPage pdPage = document.getPage(page - 1);
             PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
 
-            for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+            for (Section paragraph : classifiedDoc.getSections()) {
 
                 for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -364,7 +364,7 @@ public class RedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");
+        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1111,7 +1111,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
+        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -131,8 +131,8 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(table.getColCount()).isEqualTo(6);
         assertThat(table.getRowCount()).isEqualTo(13);
         assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@@ -146,11 +146,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(2);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@@ -165,11 +165,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(9);
         assertThat(firstTable.getRowCount()).isEqualTo(5);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(9);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
@@ -184,11 +184,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -14,6 +14,12 @@ global Section section
 //        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
 //    end
 
+rule "-1: Find dictionary entries"
+    when
+    Section(findDictionaryEntities());
+    then
+    end
+
 
 rule "0: Add CBI_author from ai"
     when