Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Added log entries

Browse Source
This commit is contained in:
deiflaender 2022-03-28 09:26:25 +02:00
parent 9ad923c314
commit 50c2d4ffdc
2 changed files with 29 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -123,8 +123,13 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
long redactionLogStartTime = System.currentTimeMillis();
var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Getting redactionLog took: {}", System.currentTimeMillis() - redactionLogStartTime);
long textStartTime = System.currentTimeMillis();
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Getting text took: {}", System.currentTimeMillis() - textStartTime);
// not yet ready for reanalysis // not yet ready for reanalysis
if (redactionLog == null || text == null || text.getNumberOfPages() == 0) { if (redactionLog == null || text == null || text.getNumberOfPages() == 0) {
@ -133,38 +138,56 @@ public class AnalyzeService {
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
long sectionsToReanalyseStartTime = System.currentTimeMillis();
Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse() Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse()
.isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest); .isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest);
log.info("Getting sectionsToReanalyse took: {}", System.currentTimeMillis() - sectionsToReanalyseStartTime);
if (sectionsToReanalyse.isEmpty()) { if (sectionsToReanalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true); return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
} }
long nerStartTime = System.currentTimeMillis();
NerEntities nerEntities; NerEntities nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) { if (redactionServiceSettings.isNerServiceEnabled()) {
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
} else { } else {
nerEntities = NerEntities.builder().build(); nerEntities = NerEntities.builder().build();
} }
log.info("Getting ner took: {}", System.currentTimeMillis() - nerStartTime);
long reanalysisSectionsStartTime = System.currentTimeMillis();
List<SectionText> reanalysisSections = text.getSectionTexts() List<SectionText> reanalysisSections = text.getSectionTexts()
.stream() .stream()
.filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber())) .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
.collect(Collectors.toList()); .collect(Collectors.toList());
log.info("Getting sections took: {}", System.currentTimeMillis() - reanalysisSectionsStartTime);
long getRulesStartTime = System.currentTimeMillis();
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
log.info("Getting rules took: {}", System.currentTimeMillis() - getRulesStartTime);
long copyDictionaryStartTime = System.currentTimeMillis();
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Copy Dictionary took: {}", System.currentTimeMillis() - copyDictionaryStartTime);
long findEntitiesStartTime = System.currentTimeMillis();
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities); PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
log.info("Find Entities took: {}", System.currentTimeMillis() - findEntitiesStartTime);
long createRedactionLogStartTime = System.currentTimeMillis();
var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId()); var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
log.info("Create redactionLog took: {}", System.currentTimeMillis() - createRedactionLogStartTime);
long importedRedactionFilteredStartTime = System.currentTimeMillis();
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newRedactionLogEntries, false); var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newRedactionLogEntries, false);
redactionLog.getRedactionLogEntry() redactionLog.getRedactionLogEntry()
.removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType() .removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType()
.equals(IMPORTED_REDACTION_TYPE)); .equals(IMPORTED_REDACTION_TYPE));
redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries); redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
log.info("Filter imported redactions took: {}", System.currentTimeMillis() - importedRedactionFilteredStartTime);
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true); return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
} }

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -50,8 +50,12 @@ public class DictionaryService {
public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) { public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
long dictionaryStartTime = System.currentTimeMillis();
DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId); DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
log.info("dictionary update took: {}", System.currentTimeMillis() - dictionaryStartTime);
long dictionaryIncrementStartTime = System.currentTimeMillis();
Set<DictionaryIncrementValue> newValues = new HashSet<>(); Set<DictionaryIncrementValue> newValues = new HashSet<>();
List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary(); List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> { dictionaryModels.forEach(dictionaryModel -> {
@ -73,6 +77,7 @@ public class DictionaryService {
}); });
} }
log.info("dictionary update took: {}", System.currentTimeMillis() - dictionaryIncrementStartTime);
return new DictionaryIncrement(newValues, version); return new DictionaryIncrement(newValues, version);
} }