Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6093: Prototype find entities in rules

Browse Source
This commit is contained in:
deiflaender 2023-02-13 15:49:20 +01:00
parent 1fca62f578
commit dd1b838a5c
14 changed files with 193 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java
View File

@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
public class Document {
private List<Page> pages = new ArrayList<>();
private List<Paragraph> paragraphs = new ArrayList<>();
private List<Section> sections = new ArrayList<>();
private List<Header> headers = new ArrayList<>();
private List<Footer> footers = new ArrayList<>();
private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Paragraph.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Section.java
View File

@ -1,7 +1,13 @@
package com.iqser.red.service.redaction.v1.server.classification.model;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@ -10,10 +16,11 @@ import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@Data
@NoArgsConstructor
public class Paragraph implements Comparable {
public class Section implements Comparable {
private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
private List<PdfImage> images = new ArrayList<>();
@ -62,4 +69,9 @@ public class Paragraph implements Comparable {
return 0;
}
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java
View File

@ -187,6 +187,11 @@ public class SearchableText {
}
// public List<String> getParagraphStrings(){
//
// }
public static String buildString(List<TextPositionSequence> sequences) {
StringBuilder sb = new StringBuilder();

104
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -18,21 +18,28 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.ArgumentType;
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.SectionArea;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import io.micrometer.core.annotation.Timed;
import lombok.Builder;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@ -85,6 +92,11 @@ public class Section {
private boolean isInTable;
@Builder.Default
private List<Integer> cellStarts = new ArrayList<>();
private RedactionServiceSettings redactionServiceSettings;
@Deprecated
@SuppressWarnings("unused")
@ -1675,6 +1687,98 @@ public class Section {
}
public boolean findDictionaryEntities(){
findEntities();
if (cellStarts != null && !cellStarts.isEmpty()) {
SurroundingWordsService.addSurroundingText(entities, searchableText, dictionary, cellStarts, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
} else {
SurroundingWordsService.addSurroundingText(entities, searchableText, dictionary, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
}
if (!isLocal && manualRedactions != null) {
var approvedForceRedactions = manualRedactions
.getForceRedactions()
.stream()
.filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
.filter(fr -> fr.getRequestDate() != null)
.collect(Collectors.toList());
// only approved id removals, that haven't been forced back afterwards
var idsToRemove = manualRedactions
.getIdsToRemove()
.stream()
.filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
.filter(idr -> idr.getRequestDate() != null)
.filter(idr -> approvedForceRedactions.stream()
.noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
.isAfter(idr.getRequestDate())))
.map(IdRemoval::getAnnotationId)
.collect(Collectors.toSet());
if (images != null && !images.isEmpty() && manualRedactions.getImageRecategorization() != null) {
for (Image image : images) {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : manualRedactions.getImageRecategorization()) {
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
image.setType(imageRecategorization.getType());
}
}
if (idsToRemove.contains(imageId)) {
image.setIgnored(true);
}
}
}
entities.forEach(entity -> entity.getPositionSequences().forEach(ps -> {
if (idsToRemove.contains(ps.getId())) {
entity.setIgnored(true);
}
}));
}
return false;
}
private void findEntities() {
Set<Entity> found = new HashSet<>();
String searchableString = searchableText.asString();
if (StringUtils.isEmpty(searchableString)) {
entities = new HashSet<>();
return;
}
String lowercaseInputString = searchableString.toLowerCase();
for (DictionaryModel model : dictionary.getDictionaryModels()) {
var searchImplementation = isLocal ? model.getLocalSearch() : model.getEntriesSearch();
var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
searchImplementation,
model,
new FindEntityDetails(model.getType(),
headline,
sectionNumber,
!isLocal,
model.isDossierDictionary(),
isLocal ? Engine.RULE : Engine.DICTIONARY,
isLocal ? EntityType.RECOMMENDATION : EntityType.ENTITY));
EntitySearchUtils.addOrAddEngine(found, entities);
}
entities = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
nerEntities = EntitySearchUtils.clearAndFindPositions(nerEntities, searchableText, dictionary, manualRedactions);
}
}

70
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -35,7 +35,6 @@ public class EntityRedactionService {
private final RedactionServiceSettings redactionServiceSettings;
private final DroolsExecutionService droolsExecutionService;
private final SurroundingWordsService surroundingWordsService;
public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@ -78,60 +77,9 @@ public class EntityRedactionService {
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
for (SectionText reanalysisSection : reanalysisSections) {
Entities entities = findEntities(reanalysisSection.getSearchableText(),
reanalysisSection.getHeadline(),
reanalysisSection.getSectionNumber(),
dictionary,
local,
nerEntities,
reanalysisSection.getCellStarts(),
analyzeRequest.getManualRedactions());
if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection.getCellStarts());
} else {
surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
}
if (!local && analyzeRequest.getManualRedactions() != null) {
var approvedForceRedactions = analyzeRequest.getManualRedactions()
.getForceRedactions()
.stream()
.filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
.filter(fr -> fr.getRequestDate() != null)
.collect(Collectors.toList());
// only approved id removals, that haven't been forced back afterwards
var idsToRemove = analyzeRequest.getManualRedactions()
.getIdsToRemove()
.stream()
.filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
.filter(idr -> idr.getRequestDate() != null)
.filter(idr -> approvedForceRedactions.stream()
.noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
.isAfter(idr.getRequestDate())))
.map(IdRemoval::getAnnotationId)
.collect(Collectors.toSet());
if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
for (Image image : reanalysisSection.getImages()) {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions().getImageRecategorization()) {
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
image.setType(imageRecategorization.getType());
}
}
if (idsToRemove.contains(imageId)) {
image.setIgnored(true);
}
}
}
entities.getEntities().forEach(entity -> entity.getPositionSequences().forEach(ps -> {
if (idsToRemove.contains(ps.getId())) {
entity.setIgnored(true);
}
}));
Set<Entity> nerFound = new HashSet<>();
if (!local) {
nerFound.addAll(getNerValues(reanalysisSection.getSectionNumber(), nerEntities, reanalysisSection.getCellStarts(), reanalysisSection.getHeadline()));
}
log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
@ -139,9 +87,7 @@ public class EntityRedactionService {
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
.isLocal(false)
.dictionaryTypes(dictionary.getTypes())
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
.stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
.nerEntities(entities.getNerEntities())
.nerEntities(nerFound)
.text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
.searchText(reanalysisSection.getSearchableText().toString())
.headline(reanalysisSection.getHeadline())
@ -154,6 +100,8 @@ public class EntityRedactionService {
.fileAttributes(analyzeRequest.getFileAttributes())
.manualRedactions(analyzeRequest.getManualRedactions())
.isInTable(reanalysisSection.isTable())
.redactionServiceSettings(redactionServiceSettings)
.cellStarts(reanalysisSection.getCellStarts())
.build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
}
@ -183,12 +131,12 @@ public class EntityRedactionService {
.collect(Collectors.toSet());
if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
sectionSearchableTextPair.getSearchableText(),
dictionary,
sectionSearchableTextPair.getCellStarts());
sectionSearchableTextPair.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
} else {
surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
}
entities.addAll(analysedSection.getEntities());

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
@ -34,8 +35,7 @@ import lombok.extern.slf4j.Slf4j;
public class ManualRedactionSurroundingTextService {
private final RedactionStorageService redactionStorageService;
private final SurroundingWordsService surroundingWordsService;
private final RedactionServiceSettings redactionServiceSettings;
@Timed("redactmanager_surroundingTextAnalysis")
public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
@ -87,9 +87,9 @@ public class ManualRedactionSurroundingTextService {
}
if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
} else {
surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
}
return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java
View File

@ -4,7 +4,7 @@ import com.iqser.red.service.redaction.v1.model.CellRectangle;
import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@ -31,7 +31,7 @@ public class SectionGridCreatorService {
private void addSectionGrid(Document classifiedDoc, int page) {
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
for (Section paragraph : classifiedDoc.getSections()) {
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java
View File

@ -16,7 +16,7 @@ import com.iqser.red.service.redaction.v1.model.SectionArea;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
@ -40,14 +40,14 @@ public class SectionTextBuilderService {
List<SectionText> sectionTexts = new ArrayList<>();
AtomicInteger sectionNumber = new AtomicInteger(1);
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
for (Section section : classifiedDoc.getSections()) {
List<Table> tables = paragraph.getTables();
List<Table> tables = section.getTables();
for (Table table : tables) {
sectionTexts.addAll(processTablePerRow(table, sectionNumber));
sectionNumber.incrementAndGet();
}
sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph.getImages()));
sectionTexts.add(processText(section.getSearchableText(), section.getTextBlocks(), section.getHeadline(), sectionNumber, section.getImages()));
sectionNumber.incrementAndGet();
}

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java
View File

@ -7,6 +7,7 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -15,15 +16,12 @@ import java.util.List;
import java.util.Set;
@Slf4j
@Service
@RequiredArgsConstructor
@UtilityClass
public class SurroundingWordsService {
private final RedactionServiceSettings redactionServiceSettings;
@Timed("redactmanager_addSurroundingText")
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
if (entities.isEmpty()) {
return;
@ -35,7 +33,7 @@ public class SurroundingWordsService {
if (dictionary != null && dictionary.isHint(entity.getType())) {
continue;
}
findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd());
findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd(), surroundingWordsOffsetWindow, numberOfSurroundingWords);
}
} catch (Exception e) {
log.warn("Could not get surrounding text!");
@ -44,7 +42,7 @@ public class SurroundingWordsService {
@Timed("redactmanager_addSurroundingTextTables")
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts) {
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
if (entities.isEmpty()) {
return;
@ -75,7 +73,7 @@ public class SurroundingWordsService {
if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
int entityStartOffset = entity.getStart() - startOffset;
int entityEndOffset = entity.getEnd() - startOffset;
findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
findSurroundingWords(entity, text, entityStartOffset, entityEndOffset, surroundingWordsOffsetWindow, numberOfSurroundingWords);
}
}
}
@ -86,23 +84,23 @@ public class SurroundingWordsService {
}
private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow();
int offsetBefore = entityStartOffset - surroundingWordsOffsetWindow < 0 ? 0 : entityStartOffset - surroundingWordsOffsetWindow;
String textBefore = text.substring(offsetBefore, entityStartOffset);
if (!textBefore.isBlank()) {
String[] wordsBefore = textBefore.split(" ");
int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsBefore.length;
int numberOfWordsBefore = wordsBefore.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsBefore.length;
if (wordsBefore.length > 0) {
entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
}
}
int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
int endOffset = entityEndOffset + surroundingWordsOffsetWindow > text.length() ? text.length() : entityEndOffset + surroundingWordsOffsetWindow;
String textAfter = text.substring(entityEndOffset, endOffset);
if (!textAfter.isBlank()) {
String[] wordsAfter = textAfter.split(" ");
int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsAfter.length;
int numberOfWordsAfter = wordsAfter.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsAfter.length;
if (wordsAfter.length > 0) {
entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
}

61
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java
View File

@ -15,7 +15,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@ -32,7 +32,7 @@ public class SectionsBuilderService {
public void buildSections(Document document) {
List<AbstractTextContainer> chunkWords = new ArrayList<>();
List<Paragraph> chunkBlockList = new ArrayList<>();
List<Section> chunkBlockList = new ArrayList<>();
List<Header> headers = new ArrayList<>();
List<Footer> footers = new ArrayList<>();
List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
@ -69,7 +69,7 @@ public class SectionsBuilderService {
}
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
chunkBlock.setHeadline(lastHeadline);
if (document.isHeadlines()) {
lastHeadline = current.getText();
@ -101,11 +101,11 @@ public class SectionsBuilderService {
}
}
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
chunkBlock.setHeadline(lastHeadline);
chunkBlockList.add(chunkBlock);
document.setParagraphs(chunkBlockList);
document.setSections(chunkBlockList);
document.setHeaders(headers);
document.setFooters(footers);
document.setUnclassifiedTexts(unclassifiedTexts);
@ -114,8 +114,8 @@ public class SectionsBuilderService {
public void addImagesToSections(Document document) {
Map<Integer, List<Paragraph>> paragraphMap = new HashMap<>();
for (Paragraph paragraph : document.getParagraphs()) {
Map<Integer, List<Section>> paragraphMap = new HashMap<>();
for (Section paragraph : document.getSections()) {
for (AbstractTextContainer container : paragraph.getPageBlocks()) {
paragraphMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()).add(paragraph);
@ -124,22 +124,22 @@ public class SectionsBuilderService {
}
if (paragraphMap.isEmpty()) {
Paragraph paragraph = new Paragraph();
document.getParagraphs().add(paragraph);
Section paragraph = new Section();
document.getSections().add(paragraph);
paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
}
// first page is always a paragraph, else we can't process pages 1..N,
// where N is the first found page with a paragraph
if (paragraphMap.get(1) == null) {
Paragraph paragraph = new Paragraph();
document.getParagraphs().add(paragraph);
Section paragraph = new Section();
document.getSections().add(paragraph);
paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
}
for (Page page : document.getPages()) {
for (PdfImage image : page.getImages()) {
List<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
List<Section> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
if (paragraphsOnPage == null) {
int i = page.getPageNumber();
while (paragraphsOnPage == null) {
@ -147,7 +147,7 @@ public class SectionsBuilderService {
i--;
}
}
for (Paragraph paragraph : paragraphsOnPage) {
for (Section paragraph : paragraphsOnPage) {
Float xMin = null;
Float yMin = null;
Float xMax = null;
@ -239,9 +239,9 @@ public class SectionsBuilderService {
}
private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
private Section buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
Paragraph paragraph = new Paragraph();
Section section = new Section();
TextBlock textBlock = null;
int pageBefore = -1;
@ -268,40 +268,17 @@ public class SectionsBuilderService {
}
if (textBlock != null && !alreadyAdded) {
paragraph.getPageBlocks().add(textBlock);
section.getPageBlocks().add(textBlock);
alreadyAdded = true;
}
paragraph.getPageBlocks().add(table);
section.getPageBlocks().add(table);
continue;
}
TextBlock wordBlock = (TextBlock) container;
if (textBlock == null) {
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
textBlock.setPage(wordBlock.getPage());
} else if (splitByTable) {
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
textBlock.setPage(wordBlock.getPage());
alreadyAdded = false;
} else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
textBlock.setPage(pageBefore);
paragraph.getPageBlocks().add(textBlock);
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
textBlock.setPage(wordBlock.getPage());
} else {
TextBlock spatialEntity = textBlock.union(wordBlock);
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
}
pageBefore = wordBlock.getPage();
splitByTable = false;
previous = container;
section.getPageBlocks().add(wordBlock);
}
if (textBlock != null && !alreadyAdded) {
paragraph.getPageBlocks().add(textBlock);
}
return paragraph;
return section;
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java
View File

@ -12,7 +12,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@ -37,7 +37,7 @@ public class PdfVisualisationService {
PDPage pdPage = document.getPage(page - 1);
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
for (Section paragraph : classifiedDoc.getSections()) {
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -364,7 +364,7 @@ public class RedactionIntegrationTest {
@Test
public void titleExtraction() throws IOException {
AnalyzeRequest request = prepareStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request);
@ -1111,7 +1111,7 @@ public class RedactionIntegrationTest {
System.out.println("classificationTest");
AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
RedactionRequest redactionRequest = RedactionRequest.builder()
.dossierId(request.getDossierId())

22
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java
View File

@ -131,8 +131,8 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(table.getColCount()).isEqualTo(6);
assertThat(table.getRowCount()).isEqualTo(13);
assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@ -146,11 +146,11 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(firstTable.getColCount()).isEqualTo(8);
assertThat(firstTable.getRowCount()).isEqualTo(1);
Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
assertThat(secondTable.getColCount()).isEqualTo(8);
assertThat(secondTable.getRowCount()).isEqualTo(2);
List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@ -165,11 +165,11 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(firstTable.getColCount()).isEqualTo(9);
assertThat(firstTable.getRowCount()).isEqualTo(5);
Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
assertThat(secondTable.getColCount()).isEqualTo(9);
assertThat(secondTable.getRowCount()).isEqualTo(6);
List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
@ -184,11 +184,11 @@ public class PdfSegmentationServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
assertThat(firstTable.getColCount()).isEqualTo(8);
assertThat(firstTable.getRowCount()).isEqualTo(1);
Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
assertThat(secondTable.getColCount()).isEqualTo(8);
assertThat(secondTable.getRowCount()).isEqualTo(6);
List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -14,6 +14,12 @@ global Section section
// section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
// end
rule "-1: Find dictionary entries"
when
Section(findDictionaryEntities());
then
end
rule "0: Add CBI_author from ai"
when