Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Add more logs

Browse Source
This commit is contained in:
lmaldacker 2021-02-09 14:06:27 +01:00
parent 9ceba4abf3
commit 3f3256469c
8 changed files with 47 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -52,6 +52,7 @@ public class RedactionController implements RedactionResource {
public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
log.info("Starting redaction analysis...");
long startTime = System.currentTimeMillis();
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
pdDocument.setAllSecurityToBeRemoved(true);
@ -66,6 +67,8 @@ public class RedactionController implements RedactionResource {
log.info("Redaction analysis successful...");
log.debug("Finishing redaction analysis took: {}.", (System.currentTimeMillis() - startTime));
return AnalyzeResult.builder()
.sectionGrid(classifiedDoc.getSectionGrid())
.redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java
View File

@ -41,7 +41,10 @@ public class AnnotationService {
public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
log.info("Annotating document.");
log.info("Annotating document...");
long startTime = System.currentTimeMillis();
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
for (int page = 1; page <= document.getNumberOfPages(); page++) {
@ -58,7 +61,7 @@ public class AnnotationService {
addAnnotations(logEntries, pdPage, page);
}
}
log.info("Finished document annotation.");
log.info("Finishing document annotation took: {}.", (System.currentTimeMillis() - startTime));
}

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -50,7 +50,8 @@ public class EntityRedactionService {
public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
log.info("Processing document.");
log.info("Processing document...");
long startTime = System.currentTimeMillis();
dictionaryService.updateDictionary(ruleSetId);
KieContainer container = droolsExecutionService.updateRules(ruleSetId);
@ -58,8 +59,14 @@ public class EntityRedactionService {
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
long current = System.currentTimeMillis() - startTime;
log.info("Adding image entries took: {}.", current);
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
current = System.currentTimeMillis() - startTime - current;
log.info("Finding entries took: {}.", current);
if (dictionary.hasLocalEntries()) {
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
@ -73,6 +80,9 @@ public class EntityRedactionService {
Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
current = System.currentTimeMillis() - startTime - current;
log.info("Adding local dictionary annotations took: {}.", current);
}
for (Entity entity : documentEntities) {
@ -91,12 +101,18 @@ public class EntityRedactionService {
}
}
current = System.currentTimeMillis() - startTime - current;
log.info("Adding entity position sequence took: {}.", current);
dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
classifiedDoc.setDictionaryVersion(dictionary.getVersion());
classifiedDoc.setRulesVersion(rulesVersion);
log.info("Finished document processing.");
current = System.currentTimeMillis() - startTime - current;
log.info("Adjusting rules set took: {}.", current);
log.info("Finishing document processing took: {}.", (System.currentTimeMillis() - startTime));
}

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -49,7 +49,8 @@ public class RedactionLogCreatorService {
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
String ruleSetId) {
log.info("Creating redaction log.");
log.info("Creating redaction log...");
long startTime = System.currentTimeMillis();
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
@ -61,16 +62,25 @@ public class RedactionLogCreatorService {
addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
}
long current = System.currentTimeMillis() - startTime;
log.info("Adding entries took: {}.", current);
if (manualRedactionPages.contains(page)) {
addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
}
current = System.currentTimeMillis() - startTime - current;
log.info("Adding manual entries took: {}.", current);
if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
addImageEntries(classifiedDoc, page, ruleSetId);
}
current = System.currentTimeMillis() - startTime - current;
log.info("Adding image entries took: {}.", current);
}
log.info("Finished redaction log creation.");
log.info("Finishing redaction log creation took: {}.", (System.currentTimeMillis() - startTime));
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java
View File

@ -39,7 +39,7 @@ public class PdfSegmentationService {
public Document parseDocument(PDDocument pdDocument) throws IOException {
log.info("Parsing document.");
log.info("Parsing document...");
Document document = new Document();
@ -93,7 +93,7 @@ public class PdfSegmentationService {
sectionsBuilderService.buildSections(document);
log.info("Finished document parsing.");
log.info("Finishing document parsing took: {}.", (System.currentTimeMillis() - startTime));
return document;
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java
View File

@ -27,7 +27,8 @@ public class SectionsBuilderService {
public void buildSections(Document document) {
log.info("Building sections.");
log.debug("Building sections...");
long startTime = System.currentTimeMillis();
List<AbstractTextContainer> chunkWords = new ArrayList<>();
List<Paragraph> chunkBlockList = new ArrayList<>();
@ -93,7 +94,7 @@ public class SectionsBuilderService {
document.setHeaders(headers);
document.setFooters(footers);
log.info("Finished section building.");
log.debug("Finishing section building took: {}.", (System.currentTimeMillis() - startTime));
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java
View File

@ -23,7 +23,7 @@ public class RulingCleaningService {
public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
log.info("Getting clean rulings.");
log.debug("Getting clean rulings...");
if (!rulings.isEmpty()) {
snapPoints(rulings, minCharWidth, maxCharHeight);

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java
View File

@ -31,7 +31,8 @@ public class TableExtractionService {
public void extractTables(CleanRulings cleanRulings, Page page) {
log.info("Extracting tables.");
log.debug("Extracting tables...");
long startTime = System.currentTimeMillis();
List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
@ -84,7 +85,7 @@ public class TableExtractionService {
page.getTextBlocks().removeAll(toBeRemoved);
log.info("Finished table extraction.");
log.debug("Finishing table extraction took: {}.", (System.currentTimeMillis() - startTime));
}