Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Add more logs

Browse Source
This commit is contained in:
lmaldacker 2021-02-09 14:06:27 +01:00
parent 9ceba4abf3
commit 3f3256469c
8 changed files with 47 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -52,6 +52,7 @@ public class RedactionController implements RedactionResource {
public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) { public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
log.info("Starting redaction analysis..."); log.info("Starting redaction analysis...");
long startTime = System.currentTimeMillis();
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) { try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
pdDocument.setAllSecurityToBeRemoved(true); pdDocument.setAllSecurityToBeRemoved(true);
@ -66,6 +67,8 @@ public class RedactionController implements RedactionResource {
log.info("Redaction analysis successful..."); log.info("Redaction analysis successful...");
log.debug("Finishing redaction analysis took: {}.", (System.currentTimeMillis() - startTime));
return AnalyzeResult.builder() return AnalyzeResult.builder()
.sectionGrid(classifiedDoc.getSectionGrid()) .sectionGrid(classifiedDoc.getSectionGrid())
.redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc .redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java
View File

@ -41,7 +41,10 @@ public class AnnotationService {
public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException { public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
log.info("Annotating document.");
log.info("Annotating document...");
long startTime = System.currentTimeMillis();
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog); Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
for (int page = 1; page <= document.getNumberOfPages(); page++) { for (int page = 1; page <= document.getNumberOfPages(); page++) {
@ -58,7 +61,7 @@ public class AnnotationService {
addAnnotations(logEntries, pdPage, page); addAnnotations(logEntries, pdPage, page);
} }
} }
log.info("Finished document annotation."); log.info("Finishing document annotation took: {}.", (System.currentTimeMillis() - startTime));
} }

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -50,7 +50,8 @@ public class EntityRedactionService {
public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) { public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
log.info("Processing document."); log.info("Processing document...");
long startTime = System.currentTimeMillis();
dictionaryService.updateDictionary(ruleSetId); dictionaryService.updateDictionary(ruleSetId);
KieContainer container = droolsExecutionService.updateRules(ruleSetId); KieContainer container = droolsExecutionService.updateRules(ruleSetId);
@ -58,8 +59,14 @@ public class EntityRedactionService {
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
long current = System.currentTimeMillis() - startTime;
log.info("Adding image entries took: {}.", current);
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null)); Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
current = System.currentTimeMillis() - startTime - current;
log.info("Finding entries took: {}.", current);
if (dictionary.hasLocalEntries()) { if (dictionary.hasLocalEntries()) {
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>(); Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
@ -73,6 +80,9 @@ public class EntityRedactionService {
Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber); Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary); EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities); EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
current = System.currentTimeMillis() - startTime - current;
log.info("Adding local dictionary annotations took: {}.", current);
} }
for (Entity entity : documentEntities) { for (Entity entity : documentEntities) {
@ -91,12 +101,18 @@ public class EntityRedactionService {
} }
} }
current = System.currentTimeMillis() - startTime - current;
log.info("Adding entity position sequence took: {}.", current);
dictionaryService.updateExternalDictionary(dictionary, ruleSetId); dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
classifiedDoc.setDictionaryVersion(dictionary.getVersion()); classifiedDoc.setDictionaryVersion(dictionary.getVersion());
classifiedDoc.setRulesVersion(rulesVersion); classifiedDoc.setRulesVersion(rulesVersion);
log.info("Finished document processing."); current = System.currentTimeMillis() - startTime - current;
log.info("Adjusting rules set took: {}.", current);
log.info("Finishing document processing took: {}.", (System.currentTimeMillis() - startTime));
} }

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -49,7 +49,8 @@ public class RedactionLogCreatorService {
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions, public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
String ruleSetId) { String ruleSetId) {
log.info("Creating redaction log."); log.info("Creating redaction log...");
long startTime = System.currentTimeMillis();
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions); Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
@ -61,16 +62,25 @@ public class RedactionLogCreatorService {
addEntries(classifiedDoc, manualRedactions, page, ruleSetId); addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
} }
long current = System.currentTimeMillis() - startTime;
log.info("Adding entries took: {}.", current);
if (manualRedactionPages.contains(page)) { if (manualRedactionPages.contains(page)) {
addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId); addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
} }
current = System.currentTimeMillis() - startTime - current;
log.info("Adding manual entries took: {}.", current);
if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) { if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
addImageEntries(classifiedDoc, page, ruleSetId); addImageEntries(classifiedDoc, page, ruleSetId);
} }
current = System.currentTimeMillis() - startTime - current;
log.info("Adding image entries took: {}.", current);
} }
log.info("Finished redaction log creation."); log.info("Finishing redaction log creation took: {}.", (System.currentTimeMillis() - startTime));
} }

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java
View File

@ -39,7 +39,7 @@ public class PdfSegmentationService {
public Document parseDocument(PDDocument pdDocument) throws IOException { public Document parseDocument(PDDocument pdDocument) throws IOException {
log.info("Parsing document."); log.info("Parsing document...");
Document document = new Document(); Document document = new Document();
@ -93,7 +93,7 @@ public class PdfSegmentationService {
sectionsBuilderService.buildSections(document); sectionsBuilderService.buildSections(document);
log.info("Finished document parsing."); log.info("Finishing document parsing took: {}.", (System.currentTimeMillis() - startTime));
return document; return document;
} }

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java
View File

@ -27,7 +27,8 @@ public class SectionsBuilderService {
public void buildSections(Document document) { public void buildSections(Document document) {
log.info("Building sections."); log.debug("Building sections...");
long startTime = System.currentTimeMillis();
List<AbstractTextContainer> chunkWords = new ArrayList<>(); List<AbstractTextContainer> chunkWords = new ArrayList<>();
List<Paragraph> chunkBlockList = new ArrayList<>(); List<Paragraph> chunkBlockList = new ArrayList<>();
@ -93,7 +94,7 @@ public class SectionsBuilderService {
document.setHeaders(headers); document.setHeaders(headers);
document.setFooters(footers); document.setFooters(footers);
log.info("Finished section building."); log.debug("Finishing section building took: {}.", (System.currentTimeMillis() - startTime));
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java
View File

@ -23,7 +23,7 @@ public class RulingCleaningService {
public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float maxCharHeight) { public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
log.info("Getting clean rulings."); log.debug("Getting clean rulings...");
if (!rulings.isEmpty()) { if (!rulings.isEmpty()) {
snapPoints(rulings, minCharWidth, maxCharHeight); snapPoints(rulings, minCharWidth, maxCharHeight);

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java
View File

@ -31,7 +31,8 @@ public class TableExtractionService {
public void extractTables(CleanRulings cleanRulings, Page page) { public void extractTables(CleanRulings cleanRulings, Page page) {
log.info("Extracting tables."); log.debug("Extracting tables...");
long startTime = System.currentTimeMillis();
List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical()); List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
@ -84,7 +85,7 @@ public class TableExtractionService {
page.getTextBlocks().removeAll(toBeRemoved); page.getTextBlocks().removeAll(toBeRemoved);
log.info("Finished table extraction."); log.debug("Finishing table extraction took: {}.", (System.currentTimeMillis() - startTime));
} }