From 3f3256469ce86f6a7ab656c5bc04c7d9bdb1180f Mon Sep 17 00:00:00 2001 From: lmaldacker Date: Tue, 9 Feb 2021 14:06:27 +0100 Subject: [PATCH] Add more logs --- .../controller/RedactionController.java | 3 +++ .../redaction/service/AnnotationService.java | 7 +++++-- .../service/EntityRedactionService.java | 20 +++++++++++++++++-- .../service/RedactionLogCreatorService.java | 14 +++++++++++-- .../segmentation/PdfSegmentationService.java | 4 ++-- .../segmentation/SectionsBuilderService.java | 5 +++-- .../service/RulingCleaningService.java | 2 +- .../service/TableExtractionService.java | 5 +++-- 8 files changed, 47 insertions(+), 13 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 8ce9511d..0d7212b6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -52,6 +52,7 @@ public class RedactionController implements RedactionResource { public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) { log.info("Starting redaction analysis..."); + long startTime = System.currentTimeMillis(); try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) { pdDocument.setAllSecurityToBeRemoved(true); @@ -66,6 +67,8 @@ public class RedactionController implements RedactionResource { log.info("Redaction analysis successful..."); + log.debug("Finishing redaction analysis took: {}.", (System.currentTimeMillis() - startTime)); + return AnalyzeResult.builder() .sectionGrid(classifiedDoc.getSectionGrid()) .redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java index b51faf66..96403b9a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java @@ -41,7 +41,10 @@ public class AnnotationService { public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException { - log.info("Annotating document."); + + log.info("Annotating document..."); + long startTime = System.currentTimeMillis(); + Map> redactionLogPerPage = convertRedactionLog(redactionLog); for (int page = 1; page <= document.getNumberOfPages(); page++) { @@ -58,7 +61,7 @@ public class AnnotationService { addAnnotations(logEntries, pdPage, page); } } - log.info("Finished document annotation."); + log.info("Finishing document annotation took: {}.", (System.currentTimeMillis() - startTime)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index adeb96a4..36a26c38 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -50,7 +50,8 @@ public class EntityRedactionService { public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) { - log.info("Processing document."); + log.info("Processing document..."); + long startTime = System.currentTimeMillis(); dictionaryService.updateDictionary(ruleSetId); KieContainer container = droolsExecutionService.updateRules(ruleSetId); @@ -58,8 +59,14 @@ public class EntityRedactionService { Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId); + long current = System.currentTimeMillis() - startTime; + log.info("Adding image entries took: {}.", current); + Set documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null)); + current = System.currentTimeMillis() - startTime - current; + log.info("Finding entries took: {}.", current); + if (dictionary.hasLocalEntries()) { Map> hintsPerSectionNumber = new HashMap<>(); @@ -73,6 +80,9 @@ public class EntityRedactionService { Set foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber); EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary); EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities); + + current = System.currentTimeMillis() - startTime - current; + log.info("Adding local dictionary annotations took: {}.", current); } for (Entity entity : documentEntities) { @@ -91,12 +101,18 @@ public class EntityRedactionService { } } + current = System.currentTimeMillis() - startTime - current; + log.info("Adding entity position sequence took: {}.", current); + dictionaryService.updateExternalDictionary(dictionary, ruleSetId); classifiedDoc.setDictionaryVersion(dictionary.getVersion()); classifiedDoc.setRulesVersion(rulesVersion); - log.info("Finished document processing."); + current = System.currentTimeMillis() - startTime - current; + log.info("Adjusting rules set took: {}.", current); + + log.info("Finishing document processing took: {}.", (System.currentTimeMillis() - startTime)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 5e01abd6..436afe64 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -49,7 +49,8 @@ public class RedactionLogCreatorService { public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions, String ruleSetId) { - log.info("Creating redaction log."); + log.info("Creating redaction log..."); + long startTime = System.currentTimeMillis(); Set manualRedactionPages = getManualRedactionPages(manualRedactions); @@ -61,16 +62,25 @@ public class RedactionLogCreatorService { addEntries(classifiedDoc, manualRedactions, page, ruleSetId); } + long current = System.currentTimeMillis() - startTime; + log.info("Adding entries took: {}.", current); + if (manualRedactionPages.contains(page)) { addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId); } + current = System.currentTimeMillis() - startTime - current; + log.info("Adding manual entries took: {}.", current); + if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) { addImageEntries(classifiedDoc, page, ruleSetId); } + + current = System.currentTimeMillis() - startTime - current; + log.info("Adding image entries took: {}.", current); } - log.info("Finished redaction log creation."); + log.info("Finishing redaction log creation took: {}.", (System.currentTimeMillis() - startTime)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java index 8b38961b..abeb065f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java @@ -39,7 +39,7 @@ public class PdfSegmentationService { public Document parseDocument(PDDocument pdDocument) throws IOException { - log.info("Parsing document."); + log.info("Parsing document..."); Document document = new Document(); @@ -93,7 +93,7 @@ public class PdfSegmentationService { sectionsBuilderService.buildSections(document); - log.info("Finished document parsing."); + log.info("Finishing document parsing took: {}.", (System.currentTimeMillis() - startTime)); return document; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java index d5590351..5e1c0c7d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java @@ -27,7 +27,8 @@ public class SectionsBuilderService { public void buildSections(Document document) { - log.info("Building sections."); + log.debug("Building sections..."); + long startTime = System.currentTimeMillis(); List chunkWords = new ArrayList<>(); List chunkBlockList = new ArrayList<>(); @@ -93,7 +94,7 @@ public class SectionsBuilderService { document.setHeaders(headers); document.setFooters(footers); - log.info("Finished section building."); + log.debug("Finishing section building took: {}.", (System.currentTimeMillis() - startTime)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java index 5b94ec4a..cf87924b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java @@ -23,7 +23,7 @@ public class RulingCleaningService { public CleanRulings getCleanRulings(List rulings, float minCharWidth, float maxCharHeight) { - log.info("Getting clean rulings."); + log.debug("Getting clean rulings..."); if (!rulings.isEmpty()) { snapPoints(rulings, minCharWidth, maxCharHeight); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java index b29e657c..4d877255 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java @@ -31,7 +31,8 @@ public class TableExtractionService { public void extractTables(CleanRulings cleanRulings, Page page) { - log.info("Extracting tables."); + log.debug("Extracting tables..."); + long startTime = System.currentTimeMillis(); List cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical()); @@ -84,7 +85,7 @@ public class TableExtractionService { page.getTextBlocks().removeAll(toBeRemoved); - log.info("Finished table extraction."); + log.debug("Finishing table extraction took: {}.", (System.currentTimeMillis() - startTime)); }