diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index 66de13e6..ec85e6e2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -1,13 +1,12 @@ package com.iqser.red.service.redaction.v1.server.queue; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL; -import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE; +import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -17,7 +16,6 @@ import org.springframework.amqp.rabbit.annotation.RabbitHandler; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; @@ -52,7 +50,12 @@ public class RedactionMessageReceiver { var errorMessage = String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(errorMessage, priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); + fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + new FileErrorInfo(errorMessage, + priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, + "redaction-service", + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); throw new AmqpRejectAndDontRequeueException(errorMessage); } @@ -70,21 +73,33 @@ public class RedactionMessageReceiver { switch (analyzeRequest.getMessageType()) { case REANALYSE: + log.info("------------------------------Reanalysis-------------------------------------"); + log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug(analyzeRequest.getManualRedactions().toString()); result = analyzeService.reanalyze(analyzeRequest); log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); + log.info("----------------------------------------------------------------"); break; case STRUCTURE_ANALYSE: + log.info("------------------------------Structure Analysis----------------------------------"); + log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); + log.info(""); break; case ANALYSE: + log.info("------------------------------Full Analysis------------------------------------"); + log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug(analyzeRequest.getManualRedactions().toString()); result = analyzeService.analyze(analyzeRequest); log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); + log.info("----------------------------------------------------------------"); break; case SURROUNDING_TEXT: + log.info("------------------------------Add surrounding Text--------------------------------"); result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions()); @@ -92,6 +107,7 @@ public class RedactionMessageReceiver { analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); + log.info("----------------------------------------------------------------"); break; default: @@ -104,7 +120,9 @@ public class RedactionMessageReceiver { } catch (Exception e) { log.warn("Failed to process analyze request: {}", analyzeRequest, e); var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); - fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); + fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); } } @@ -119,7 +137,9 @@ public class RedactionMessageReceiver { OffsetDateTime timestamp = in.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER); timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); log.info("Failed to process analyze request, errorCause: {}, timestamp: {}", errorCause, timestamp); - fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp)); + fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp)); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index a8b74763..6ebbf6d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -84,7 +84,6 @@ public class AnalyzeService { @Timed("redactmanager_analyzeDocumentStructure") public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) { - log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); long startTime = System.currentTimeMillis(); ClassificationDocument classifiedDoc; @@ -93,29 +92,29 @@ public class AnalyzeService { var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN)); - log.info("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Map> pdfImages = null; if (redactionServiceSettings.isEnableImageClassification()) { pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - log.info("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages); - log.info("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } catch (Exception e) { throw new RedactionException(e); } Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc); - log.info("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document); - log.info("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, sectionGrid); - log.info("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); return AnalyzeResult.builder() .dossierId(analyzeRequest.getDossierId()) @@ -130,27 +129,26 @@ public class AnalyzeService { @Timed("redactmanager_analyze") public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { - log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); long startTime = System.currentTimeMillis(); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); - log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - log.info("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); - log.info("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId()); entityRedactionService.addDictionaryEntities(dictionary, document); - log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities); - log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); @@ -179,12 +177,11 @@ public class AnalyzeService { @SneakyThrows public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { - log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); long startTime = System.currentTimeMillis(); RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); // not yet ready for reanalysis if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) { return analyze(analyzeRequest); @@ -196,7 +193,7 @@ public class AnalyzeService { Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement); List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); - log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (sectionsToReAnalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, @@ -209,16 +206,16 @@ public class AnalyzeService { } NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); - log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - log.info("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); - log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); - log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.debug("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java index 210e3a55..3cf977b6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java @@ -77,7 +77,6 @@ public class ManualRedactionSurroundingTextService { .stream() .map(boundary -> entityCreationService.forceByBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node)) .collect(Collectors.toSet()); - entities.forEach(RedactionEntity::removeFromGraph); RedactionEntity correctEntity = getEntityOnCorrectPosition(entities, toFindPositions); return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index 6b5e1849..d7115aae 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -1336,6 +1336,19 @@ rule "MAN.2.0: Apply force redaction" retract($force); end +rule "MAN.2.1: Apply force redaction to images" + no-loop true + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $imageToForce: Image(id == $id) + then + $imageToForce.force("MAN.2.0", "Forced redaction", $legalBasis); + $imageToForce.setRemoved(false); + $imageToForce.setIgnored(false); + update($imageToForce); + update($imageToForce.getParent()); + end // Rule unit: MAN.3 rule "MAN.3.0: Apply image recategorization" diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml index b4895cfb..2bf23009 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml @@ -10,7 +10,7 @@ - +