Compare commits

...
6 Commits
Author SHA1 Message Date
lmaldacker 0b30da89bc Fix pmd 2021-02-09 16:19:04 +01:00
lmaldacker 97abac1add Add missing var 2021-02-09 14:58:27 +01:00
lmaldacker 3f3256469c Add more logs 2021-02-09 14:06:27 +01:00
lmaldacker 9ceba4abf3 Adjust imports 2021-02-09 10:36:58 +01:00
lmaldacker ebca1fc750 Adjust pom 2021-02-09 10:21:34 +01:00
lmaldacker 37125f14fd Add logs 2021-02-08 16:22:26 +01:00
9 changed files with 84 additions and 0 deletions
@@ -10,6 +10,9 @@
</parent>
<artifactId>redaction-service-server-v1</artifactId>
<properties>
<slf4j.version>1.7.30</slf4j.version>
</properties>
<dependencies>
<dependency>
@@ -68,6 +71,11 @@
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox-tools</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<!-- spring -->
<dependency>
<groupId>org.springframework.cloud</groupId>
@@ -51,6 +51,9 @@ public class RedactionController implements RedactionResource {
@Override
public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
log.info("Starting redaction analysis...");
long startTime = System.currentTimeMillis();
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
pdDocument.setAllSecurityToBeRemoved(true);
@@ -64,6 +67,8 @@ public class RedactionController implements RedactionResource {
log.info("Redaction analysis successful...");
log.debug("Finishing redaction analysis took: {}.", System.currentTimeMillis() - startTime);
return AnalyzeResult.builder()
.sectionGrid(classifiedDoc.getSectionGrid())
.redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
@@ -30,7 +30,9 @@ import com.iqser.red.service.redaction.v1.model.SectionGrid;
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class AnnotationService {
@@ -40,6 +42,9 @@ public class AnnotationService {
public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
log.info("Annotating document...");
long startTime = System.currentTimeMillis();
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
for (int page = 1; page <= document.getNumberOfPages(); page++) {
@@ -56,6 +61,7 @@ public class AnnotationService {
addAnnotations(logEntries, pdPage, page);
}
}
log.info("Finishing document annotation took: {}.", System.currentTimeMillis() - startTime);
}
@@ -50,14 +50,23 @@ public class EntityRedactionService {
public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
log.info("Processing document...");
long startTime = System.currentTimeMillis();
dictionaryService.updateDictionary(ruleSetId);
KieContainer container = droolsExecutionService.updateRules(ruleSetId);
long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId);
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
long current = System.currentTimeMillis() - startTime;
log.info("Adding image entries took: {}.", current);
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
current = System.currentTimeMillis() - startTime - current;
log.info("Finding entries took: {}.", current);
if (dictionary.hasLocalEntries()) {
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
@@ -71,6 +80,9 @@ public class EntityRedactionService {
Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
current = System.currentTimeMillis() - startTime - current;
log.info("Adding local dictionary annotations took: {}.", current);
}
for (Entity entity : documentEntities) {
@@ -89,10 +101,19 @@ public class EntityRedactionService {
}
}
current = System.currentTimeMillis() - startTime - current;
log.info("Adding entity position sequence took: {}.", current);
dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
classifiedDoc.setDictionaryVersion(dictionary.getVersion());
classifiedDoc.setRulesVersion(rulesVersion);
current = System.currentTimeMillis() - startTime - current;
log.info("Adjusting rules set took: {}.", current);
log.info("Finishing document processing took: {}.", System.currentTimeMillis() - startTime);
}
@@ -34,7 +34,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class RedactionLogCreatorService {
@@ -47,6 +49,9 @@ public class RedactionLogCreatorService {
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
String ruleSetId) {
log.info("Creating redaction log...");
long startTime = System.currentTimeMillis();
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
for (int page = 1; page <= numberOfPages; page++) {
@@ -57,14 +62,25 @@ public class RedactionLogCreatorService {
addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
}
long current = System.currentTimeMillis() - startTime;
log.info("Adding entries took: {}.", current);
if (manualRedactionPages.contains(page)) {
addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
}
current = System.currentTimeMillis() - startTime - current;
log.info("Adding manual entries took: {}.", current);
if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
addImageEntries(classifiedDoc, page, ruleSetId);
}
current = System.currentTimeMillis() - startTime - current;
log.info("Adding image entries took: {}.", current);
}
log.info("Finishing redaction log creation took: {}.", System.currentTimeMillis() - startTime);
}
@@ -39,6 +39,9 @@ public class PdfSegmentationService {
public Document parseDocument(PDDocument pdDocument) throws IOException {
log.info("Parsing document...");
long startTime = System.currentTimeMillis();
Document document = new Document();
List<Page> pages = new ArrayList<>();
@@ -91,6 +94,8 @@ public class PdfSegmentationService {
sectionsBuilderService.buildSections(document);
log.info("Finishing document parsing took: {}.", System.currentTimeMillis() - startTime);
return document;
}
@@ -19,11 +19,17 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class SectionsBuilderService {
public void buildSections(Document document) {
log.debug("Building sections...");
long startTime = System.currentTimeMillis();
List<AbstractTextContainer> chunkWords = new ArrayList<>();
List<Paragraph> chunkBlockList = new ArrayList<>();
List<Header> headers = new ArrayList<>();
@@ -87,6 +93,9 @@ public class SectionsBuilderService {
document.setParagraphs(chunkBlockList);
document.setHeaders(headers);
document.setFooters(footers);
log.debug("Finishing section building took: {}.", System.currentTimeMillis() - startTime);
}
@@ -15,11 +15,16 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRuli
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class RulingCleaningService {
public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
log.debug("Getting clean rulings...");
if (!rulings.isEmpty()) {
snapPoints(rulings, minCharWidth, maxCharHeight);
}
@@ -23,11 +23,17 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class TableExtractionService {
public void extractTables(CleanRulings cleanRulings, Page page) {
log.debug("Extracting tables...");
long startTime = System.currentTimeMillis();
List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
List<TextBlock> toBeRemoved = new ArrayList<>();
@@ -78,6 +84,9 @@ public class TableExtractionService {
}
page.getTextBlocks().removeAll(toBeRemoved);
log.debug("Finishing table extraction took: {}.", System.currentTimeMillis() - startTime);
}