Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0b30da89bc | ||
|
|
97abac1add | ||
|
|
3f3256469c | ||
|
|
9ceba4abf3 | ||
|
|
ebca1fc750 | ||
|
|
37125f14fd | ||
|
|
6d9ed080ce | ||
|
|
00b0cb1603 | ||
|
|
5ecf21290c | ||
|
|
8965e76548 | ||
|
|
ba8c4dc240 | ||
|
|
577db37b11 | ||
|
|
154e09b843 |
@@ -10,6 +10,9 @@
|
||||
</parent>
|
||||
|
||||
<artifactId>redaction-service-server-v1</artifactId>
|
||||
<properties>
|
||||
<slf4j.version>1.7.30</slf4j.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -68,6 +71,11 @@
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox-tools</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<!-- spring -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
|
||||
+2
@@ -18,6 +18,8 @@ public class Document {
|
||||
|
||||
private List<Page> pages = new ArrayList<>();
|
||||
private List<Paragraph> paragraphs = new ArrayList<>();
|
||||
private List<Header> headers = new ArrayList<>();
|
||||
private List<Footer> footers = new ArrayList<>();
|
||||
private Map<Integer, List<Entity>> entities = new HashMap<>();
|
||||
private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
|
||||
private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Footer {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Header {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
+5
@@ -51,6 +51,9 @@ public class RedactionController implements RedactionResource {
|
||||
@Override
|
||||
public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
|
||||
|
||||
log.info("Starting redaction analysis...");
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
@@ -64,6 +67,8 @@ public class RedactionController implements RedactionResource {
|
||||
|
||||
log.info("Redaction analysis successful...");
|
||||
|
||||
log.debug("Finishing redaction analysis took: {}.", System.currentTimeMillis() - startTime);
|
||||
|
||||
return AnalyzeResult.builder()
|
||||
.sectionGrid(classifiedDoc.getSectionGrid())
|
||||
.redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
|
||||
|
||||
+29
-12
@@ -32,6 +32,7 @@ public class SearchableText {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("checkstyle:ModifiedControlVariable")
|
||||
public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive,
|
||||
List<TextPositionSequence> sequencesSubList) {
|
||||
@@ -66,9 +67,12 @@ public class SearchableText {
|
||||
for (int j = 0; j < searchSpace.get(i).length(); j++) {
|
||||
|
||||
if (i > 0 && j == 0 && searchSpace.get(i).charAt(0, caseInsensitive) == ' ' && searchSpace.get(i - 1)
|
||||
.charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == ' ' && searchSpace.get(i).charAt(j - 1, caseInsensitive) == ' ') {
|
||||
if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
|
||||
.charAt(searchSpace.get(i - 1)
|
||||
.length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == ' ' && searchSpace.get(i)
|
||||
.charAt(j - 1, caseInsensitive) == ' ') {
|
||||
if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions()
|
||||
.isEmpty()) {
|
||||
crossSequenceParts.add(partMatch);
|
||||
}
|
||||
continue;
|
||||
@@ -80,8 +84,8 @@ public class SearchableText {
|
||||
counter++;
|
||||
}
|
||||
|
||||
if (searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i)
|
||||
if (searchSpace.get(i).charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace
|
||||
.get(i)
|
||||
.charAt(j, caseInsensitive) == '-') {
|
||||
|
||||
if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
|
||||
@@ -100,14 +104,15 @@ public class SearchableText {
|
||||
if (counter == searchString.length()) {
|
||||
crossSequenceParts.add(partMatch);
|
||||
|
||||
if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
|
||||
.length() - 1 && isSeparator(searchSpace.get(i)
|
||||
if (i == searchSpace.size() - 1 && j == searchSpace.get(i)
|
||||
.length() - 1 || j != searchSpace.get(i).length() - 1 && isSeparator(searchSpace.get(i)
|
||||
.charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
|
||||
.length() - 1 && isSeparator(searchSpace.get(i + 1)
|
||||
.charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
|
||||
.charAt(0, caseInsensitive)) || j == searchSpace.get(i)
|
||||
.length() - 1 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
|
||||
.charAt(0, caseInsensitive) != ' ') {
|
||||
finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
|
||||
finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts, normalizedSearchString));
|
||||
}
|
||||
|
||||
counter = 0;
|
||||
@@ -130,15 +135,21 @@ public class SearchableText {
|
||||
}
|
||||
|
||||
return finalMatches;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {
|
||||
private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts,
|
||||
String searchString) {
|
||||
|
||||
List<EntityPositionSequence> result = new ArrayList<>();
|
||||
String asString = buildString(crossSequenceParts);
|
||||
if (!asString.equalsIgnoreCase(searchString)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
String plainId = IdBuilder.buildId(crossSequenceParts);
|
||||
String id = plainId;
|
||||
List<EntityPositionSequence> result = new ArrayList<>();
|
||||
|
||||
int currentPage = -1;
|
||||
int idDiffentPageSuffix = 1;
|
||||
EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
|
||||
@@ -173,6 +184,12 @@ public class SearchableText {
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return buildString(sequences);
|
||||
}
|
||||
|
||||
|
||||
public String buildString(List<TextPositionSequence> sequences) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
TextPositionSequence previous = null;
|
||||
|
||||
+1
-1
@@ -156,7 +156,7 @@ public class Section {
|
||||
public void addHintAnnotation(String value, String asType) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
}
|
||||
|
||||
|
||||
|
||||
+6
@@ -30,7 +30,9 @@ import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AnnotationService {
|
||||
@@ -40,6 +42,9 @@ public class AnnotationService {
|
||||
|
||||
public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
|
||||
|
||||
log.info("Annotating document...");
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
@@ -56,6 +61,7 @@ public class AnnotationService {
|
||||
addAnnotations(logEntries, pdPage, page);
|
||||
}
|
||||
}
|
||||
log.info("Finishing document annotation took: {}.", System.currentTimeMillis() - startTime);
|
||||
}
|
||||
|
||||
|
||||
|
||||
+40
-7
@@ -19,6 +19,8 @@ import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
@@ -48,14 +50,23 @@ public class EntityRedactionService {
|
||||
|
||||
public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
|
||||
|
||||
log.info("Processing document...");
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
dictionaryService.updateDictionary(ruleSetId);
|
||||
KieContainer container = droolsExecutionService.updateRules(ruleSetId);
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId);
|
||||
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
|
||||
|
||||
long current = System.currentTimeMillis() - startTime;
|
||||
log.info("Adding image entries took: {}.", current);
|
||||
|
||||
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
|
||||
|
||||
current = System.currentTimeMillis() - startTime - current;
|
||||
log.info("Finding entries took: {}.", current);
|
||||
|
||||
if (dictionary.hasLocalEntries()) {
|
||||
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
|
||||
@@ -69,6 +80,9 @@ public class EntityRedactionService {
|
||||
Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
|
||||
|
||||
current = System.currentTimeMillis() - startTime - current;
|
||||
log.info("Adding local dictionary annotations took: {}.", current);
|
||||
}
|
||||
|
||||
for (Entity entity : documentEntities) {
|
||||
@@ -87,10 +101,19 @@ public class EntityRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
current = System.currentTimeMillis() - startTime - current;
|
||||
log.info("Adding entity position sequence took: {}.", current);
|
||||
|
||||
dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
|
||||
|
||||
classifiedDoc.setDictionaryVersion(dictionary.getVersion());
|
||||
classifiedDoc.setRulesVersion(rulesVersion);
|
||||
|
||||
current = System.currentTimeMillis() - startTime - current;
|
||||
log.info("Adjusting rules set took: {}.", current);
|
||||
|
||||
log.info("Finishing document processing took: {}.", System.currentTimeMillis() - startTime);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -113,7 +136,18 @@ public class EntityRedactionService {
|
||||
}
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
sectionSearchableTextPairs.add(processText(paragraph, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
sectionSearchableTextPairs.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph
|
||||
.getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
|
||||
for (Header header : classifiedDoc.getHeaders()) {
|
||||
sectionSearchableTextPairs.add(processText(header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
|
||||
for (Footer footer : classifiedDoc.getFooters()) {
|
||||
sectionSearchableTextPairs.add(processText(footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
|
||||
@@ -253,14 +287,13 @@ public class EntityRedactionService {
|
||||
}
|
||||
|
||||
|
||||
private SectionSearchableTextPair processText(Paragraph paragraph, ManualRedactions manualRedactions,
|
||||
private SectionSearchableTextPair processText(SearchableText searchableText, List<TextBlock> paragraphTextBlocks,
|
||||
String headline, ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary, boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
SearchableText searchableText = paragraph.getSearchableText();
|
||||
addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber
|
||||
.intValue());
|
||||
Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber.intValue(), dictionary, local);
|
||||
addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue());
|
||||
Set<Entity> entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);
|
||||
|
||||
return new SectionSearchableTextPair(Section.builder()
|
||||
@@ -271,7 +304,7 @@ public class EntityRedactionService {
|
||||
.collect(Collectors.toSet()) : entities)
|
||||
.text(searchableText.getAsStringWithLinebreaks())
|
||||
.searchText(searchableText.toString())
|
||||
.headline(paragraph.getHeadline())
|
||||
.headline(headline)
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.searchableText(searchableText)
|
||||
.dictionary(dictionary)
|
||||
|
||||
+17
-1
@@ -34,7 +34,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionLogCreatorService {
|
||||
@@ -47,6 +49,9 @@ public class RedactionLogCreatorService {
|
||||
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
|
||||
String ruleSetId) {
|
||||
|
||||
log.info("Creating redaction log...");
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
|
||||
|
||||
for (int page = 1; page <= numberOfPages; page++) {
|
||||
@@ -57,14 +62,25 @@ public class RedactionLogCreatorService {
|
||||
addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
|
||||
}
|
||||
|
||||
long current = System.currentTimeMillis() - startTime;
|
||||
log.info("Adding entries took: {}.", current);
|
||||
|
||||
if (manualRedactionPages.contains(page)) {
|
||||
addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
|
||||
}
|
||||
|
||||
current = System.currentTimeMillis() - startTime - current;
|
||||
log.info("Adding manual entries took: {}.", current);
|
||||
|
||||
if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
|
||||
addImageEntries(classifiedDoc, page, ruleSetId);
|
||||
}
|
||||
|
||||
current = System.currentTimeMillis() - startTime - current;
|
||||
log.info("Adding image entries took: {}.", current);
|
||||
}
|
||||
|
||||
log.info("Finishing redaction log creation took: {}.", System.currentTimeMillis() - startTime);
|
||||
}
|
||||
|
||||
|
||||
@@ -194,7 +210,7 @@ public class RedactionLogCreatorService {
|
||||
startIndex = i;
|
||||
}
|
||||
}
|
||||
if (startIndex != textPositions.size() - 1) {
|
||||
if (startIndex != textPositions.size()) {
|
||||
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
|
||||
}
|
||||
}
|
||||
|
||||
+6
@@ -121,4 +121,10 @@ public class EntitySearchUtils {
|
||||
}
|
||||
entities.add(found);
|
||||
}
|
||||
|
||||
public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found){
|
||||
// HashSet keeps old value but we want the new.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
}
|
||||
}
|
||||
|
||||
+5
@@ -39,6 +39,9 @@ public class PdfSegmentationService {
|
||||
|
||||
public Document parseDocument(PDDocument pdDocument) throws IOException {
|
||||
|
||||
log.info("Parsing document...");
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
Document document = new Document();
|
||||
|
||||
List<Page> pages = new ArrayList<>();
|
||||
@@ -91,6 +94,8 @@ public class PdfSegmentationService {
|
||||
|
||||
sectionsBuilderService.buildSections(document);
|
||||
|
||||
log.info("Finishing document parsing took: {}.", System.currentTimeMillis() - startTime);
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
|
||||
+37
-8
@@ -10,6 +10,8 @@ import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
@@ -17,32 +19,52 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class SectionsBuilderService {
|
||||
|
||||
public void buildSections(Document document) {
|
||||
|
||||
log.debug("Building sections...");
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
List<AbstractTextContainer> chunkWords = new ArrayList<>();
|
||||
List<Paragraph> chunkBlockList = new ArrayList<>();
|
||||
List<Header> headers = new ArrayList<>();
|
||||
List<Footer> footers = new ArrayList<>();
|
||||
|
||||
AbstractTextContainer prev = null;
|
||||
|
||||
String lastHeadline = "";
|
||||
Table previousTable = null;
|
||||
for (Page page : document.getPages()) {
|
||||
List<TextBlock> header = new ArrayList<>();
|
||||
List<TextBlock> footer = new ArrayList<>();
|
||||
for (AbstractTextContainer current : page.getTextBlocks()) {
|
||||
|
||||
if (current.getClassification() == null || current.getClassification()
|
||||
.equals("Header") || current.getClassification().equals("Footer")) {
|
||||
if (current.getClassification() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
current.setPage(page.getPageNumber());
|
||||
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
if (current.getClassification().equals("Header")) {
|
||||
header.add((TextBlock) current);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.getClassification().equals("Footer")) {
|
||||
footer.add((TextBlock) current);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification()
|
||||
.startsWith("H ") || !document.isHeadlines()) {
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
chunkBlock.setHeadline(lastHeadline);
|
||||
if(document.isHeadlines()) {
|
||||
if (document.isHeadlines()) {
|
||||
lastHeadline = current.getText();
|
||||
}
|
||||
chunkBlockList.add(chunkBlock);
|
||||
@@ -60,6 +82,8 @@ public class SectionsBuilderService {
|
||||
chunkWords.add(current);
|
||||
prev = current;
|
||||
}
|
||||
headers.add(new Header(header));
|
||||
footers.add(new Footer(footer));
|
||||
}
|
||||
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
@@ -67,6 +91,11 @@ public class SectionsBuilderService {
|
||||
chunkBlockList.add(chunkBlock);
|
||||
|
||||
document.setParagraphs(chunkBlockList);
|
||||
document.setHeaders(headers);
|
||||
document.setFooters(footers);
|
||||
|
||||
log.debug("Finishing section building took: {}.", System.currentTimeMillis() - startTime);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -175,9 +204,9 @@ public class SectionsBuilderService {
|
||||
|
||||
private boolean hasInvalidHeaderInformation(Table table) {
|
||||
|
||||
return table.getRows().stream()
|
||||
.flatMap(row -> row.stream()
|
||||
.filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
|
||||
return table.getRows()
|
||||
.stream()
|
||||
.flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
|
||||
.findAny()
|
||||
.isEmpty();
|
||||
|
||||
@@ -188,7 +217,7 @@ public class SectionsBuilderService {
|
||||
|
||||
for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
|
||||
List<Cell> row = table.getRows().get(i);
|
||||
if(row.size() == 1){
|
||||
if (row.size() == 1) {
|
||||
continue;
|
||||
}
|
||||
boolean allNonHeader = true;
|
||||
|
||||
+5
@@ -15,11 +15,16 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRuli
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class RulingCleaningService {
|
||||
|
||||
public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
|
||||
|
||||
log.debug("Getting clean rulings...");
|
||||
|
||||
if (!rulings.isEmpty()) {
|
||||
snapPoints(rulings, minCharWidth, maxCharHeight);
|
||||
}
|
||||
|
||||
+9
@@ -23,11 +23,17 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class TableExtractionService {
|
||||
|
||||
public void extractTables(CleanRulings cleanRulings, Page page) {
|
||||
|
||||
log.debug("Extracting tables...");
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
|
||||
|
||||
List<TextBlock> toBeRemoved = new ArrayList<>();
|
||||
@@ -78,6 +84,9 @@ public class TableExtractionService {
|
||||
}
|
||||
|
||||
page.getTextBlocks().removeAll(toBeRemoved);
|
||||
|
||||
log.debug("Finishing table extraction took: {}.", System.currentTimeMillis() - startTime);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
+9
-1
@@ -418,6 +418,7 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
|
||||
private List<File> getPathsRecursively(File path) {
|
||||
|
||||
List<File> result = new ArrayList<>();
|
||||
@@ -439,9 +440,16 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void redactionTest() throws IOException {
|
||||
|
||||
// 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
|
||||
// 182 Fludioxonil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
|
||||
// 38 A14325E - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf
|
||||
// 91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf
|
||||
// 95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf
|
||||
|
||||
|
||||
System.out.println("redactionTest");
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
|
||||
Reference in New Issue
Block a user