From e4d44b8e17ab65840fd6c1a5308761d98c512ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Thu, 6 Jul 2023 01:39:39 +0200 Subject: [PATCH] RED-6929: fix acceptance tests/rules --- .../queue/RedactionMessageReceiver.java | 39 +++++++++++++------ .../redaction/service/AnalyzeService.java | 38 +++++++++--------- .../redaction/service/DictionaryService.java | 2 +- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index ec85e6e2..dfbf04ba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -5,6 +5,7 @@ import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfigura import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER; +import static java.lang.String.format; import java.io.IOException; import java.time.OffsetDateTime; @@ -47,7 +48,7 @@ public class RedactionMessageReceiver { // This prevents from endless retries oom errors. if (message.getMessageProperties().isRedelivered()) { - var errorMessage = String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", + var errorMessage = format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", analyzeRequest.getDossierId(), analyzeRequest.getFileId()); fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), @@ -73,41 +74,55 @@ public class RedactionMessageReceiver { switch (analyzeRequest.getMessageType()) { case REANALYSE: - log.info("------------------------------Reanalysis-------------------------------------"); + log.info(""); + log.info("------------------------------Reanalysis------------------------------------------"); log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.debug(analyzeRequest.getManualRedactions().toString()); result = analyzeService.reanalyze(analyzeRequest); - log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); - log.info("----------------------------------------------------------------"); + log.info("Successfully reanalyzed dossier {} file {} took: {} s", + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + format("%.2f", result.getDuration() / 1000.0)); + log.info("----------------------------------------------------------------------------------"); break; case STRUCTURE_ANALYSE: + log.info(""); log.info("------------------------------Structure Analysis----------------------------------"); log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); - log.info(""); + log.info("Successfully analyzed structure dossier {} file {} took: {} s", + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + format("%.2f", result.getDuration() / 1000.0)); + log.info("----------------------------------------------------------------------------------"); break; case ANALYSE: - log.info("------------------------------Full Analysis------------------------------------"); + log.info(""); + log.info("------------------------------Full Analysis---------------------------------------"); log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.debug(analyzeRequest.getManualRedactions().toString()); result = analyzeService.analyze(analyzeRequest); - log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); - log.info("----------------------------------------------------------------"); + log.info("Successfully analyzed dossier {} file {} took: {} s", + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + format("%.2f", result.getDuration() / 1000.0)); + log.info("----------------------------------------------------------------------------------"); break; case SURROUNDING_TEXT: + log.info(""); log.info("------------------------------Add surrounding Text--------------------------------"); + log.info("Starting surrounding text search for entries {} ", analyzeRequest.getManualRedactions().getEntriesToAdd()); result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions()); - log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", + log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {} s", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), - result.getDuration()); - log.info("----------------------------------------------------------------"); + format("%.2f", result.getDuration() / 1000.0)); + log.info("----------------------------------------------------------------------------------"); break; default: diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 6ebbf6d6..0f6a5beb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -92,29 +92,29 @@ public class AnalyzeService { var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN)); - log.debug("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Map> pdfImages = null; if (redactionServiceSettings.isEnableImageClassification()) { pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - log.debug("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages); - log.debug("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } catch (Exception e) { throw new RedactionException(e); } Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc); - log.debug("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document); - log.debug("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, sectionGrid); - log.debug("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); return AnalyzeResult.builder() .dossierId(analyzeRequest.getDossierId()) @@ -131,24 +131,24 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - log.debug("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); - log.debug("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - log.debug("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); - log.debug("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId()); entityRedactionService.addDictionaryEntities(dictionary, document); - log.debug("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities); - log.debug("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); @@ -179,9 +179,9 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - log.debug("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - log.debug("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); // not yet ready for reanalysis if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) { return analyze(analyzeRequest); @@ -193,7 +193,7 @@ public class AnalyzeService { Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement); List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); - log.debug("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (sectionsToReAnalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, @@ -206,16 +206,16 @@ public class AnalyzeService { } NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); - log.debug("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - log.debug("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); - log.debug("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); - log.debug("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index e2856fe5..54974fcd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -76,7 +76,7 @@ public class DictionaryService { @Timed("redactmanager_updateDictionary") public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) { - log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId); + log.debug("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId); long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId); var dossierTemplateDictionary = getDossierTemplateDictionary(dossierTemplateId); if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {