From 9ee4ae2757354290136f593c5307b29836d76b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Tue, 5 Nov 2024 14:34:43 +0100 Subject: [PATCH] RED-10353: Increase rules timeout, added different error message and error... --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../redaction-service-server-v1/build.gradle.kts | 2 +- .../v1/server/RedactionServiceSettings.java | 2 +- .../v1/server/queue/RedactionMessageReceiver.java | 14 ++++++++++++-- .../v1/server/service/AnalyzeService.java | 6 +++--- .../drools/ComponentDroolsExecutionService.java | 6 +++--- .../drools/EntityDroolsExecutionService.java | 10 +++++----- .../utils/exception/DroolsTimeoutException.java | 9 +++++++++ .../graph/DocumentPerformanceIntegrationTest.java | 2 +- .../src/test/resources/files/syngenta | 2 +- 10 files changed, 37 insertions(+), 18 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 7c8082ac..ba4dc082 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.465.79" +val persistenceServiceVersion = "2.465.83" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 333c2a96..66ec7114 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.142.6" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.465.79" +val persistenceServiceVersion = "2.465.83" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index f686f14b..c76f500b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -28,7 +28,7 @@ public class RedactionServiceSettings { private int dictionaryCacheExpireAfterAccessDays = 3; - private int droolsExecutionTimeoutSecs = 300; + private int droolsExecutionTimeoutSecs = 600; private boolean ruleExecutionSecured = true; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index 851f7dac..55f8d726 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; @@ -140,10 +141,19 @@ public class RedactionMessageReceiver { private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) { log.error("Failed to process analyze request: {}", analyzeRequest, e); - var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + ErrorCode errorCode = null; + + if(e instanceof DroolsTimeoutException dre){ + if (!dre.isReported()){ + errorCode = ErrorCode.RULES_EXECUTION_TIMEOUT; + } else { + errorCode = ErrorCode.LOCKED_RULES; + } + } + fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), - new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); + new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", errorCode)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index d4c54445..a3956514 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -155,7 +155,7 @@ public class AnalyzeService { log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); // we could add the imported redactions similar to the manual redactions here as well for additional processing - List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), + List allFileAttributes = entityDroolsExecutionService.executeRules(analyzeRequest.getFileId(), kieWrapperEntityRules.container(), document, sectionsToReAnalyse, dictionary, @@ -221,7 +221,7 @@ public class AnalyzeService { log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); // we could add the imported redactions similar to the manual redactions here as well for additional processing - List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), + List allFileAttributes = entityDroolsExecutionService.executeRules(analyzeRequest.getFileId(), kieWrapperEntityRules.container(), document, dictionary, analyzeRequest.getFileAttributes(), @@ -350,7 +350,7 @@ public class AnalyzeService { // We need the latest EntityLog entries for components rules execution entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry()); - List components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), + List components = componentDroolsExecutionService.executeRules(analyzeRequest.getFileId(), kieWrapperComponentRules.container(), entityLog, document, addedFileAttributes, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java index a8e6d4e3..ee189e83 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java @@ -49,7 +49,7 @@ public class ComponentDroolsExecutionService { ComponentMappingMemoryCache componentMappingMemoryCache; - public List executeRules(KieContainer kieContainer, + public List executeRules(String fileId, KieContainer kieContainer, EntityLog entityLog, Document document, Set fileAttributes, @@ -99,14 +99,14 @@ public class ComponentDroolsExecutionService { } catch (ExecutionException e) { kieSession.dispose(); if (e.getCause() instanceof TimeoutException) { - throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT); + throw new DroolsTimeoutException(String.format("The file %s caused a timeout", fileId), e, false, RuleFileType.COMPONENT); } throw new RuntimeException(e); } catch (InterruptedException e) { kieSession.dispose(); throw new RuntimeException(e); } catch (TimeoutException e) { - throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT); + throw new DroolsTimeoutException(String.format("The file %s caused a timeout", fileId), e, false, RuleFileType.COMPONENT); } List resultingFileAttributes = getFileAttributes(kieSession); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index 0ec900c6..3c1e8c51 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -52,14 +52,14 @@ public class EntityDroolsExecutionService { @Timed("redactmanager_executeRules") @Observed(name = "EntityDroolsExecutionService", contextualName = "execute-entity-rules") - public List executeRules(KieContainer kieContainer, + public List executeRules(String fileId, KieContainer kieContainer, Document document, Dictionary dictionary, List fileAttributes, ManualRedactions manualRedactions, NerEntities nerEntities) { - return executeRules(kieContainer, + return executeRules(fileId, kieContainer, document, document.streamChildren() .toList(), @@ -72,7 +72,7 @@ public class EntityDroolsExecutionService { @Timed("redactmanager_executeRules") @Observed(name = "EntityDroolsExecutionService", contextualName = "execute-entity-rules") - public List executeRules(KieContainer kieContainer, + public List executeRules(String fileId, KieContainer kieContainer, Document document, List sectionsToAnalyze, Dictionary dictionary, @@ -133,14 +133,14 @@ public class EntityDroolsExecutionService { } catch (ExecutionException e) { kieSession.dispose(); if (e.getCause() instanceof TimeoutException) { - throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY); + throw new DroolsTimeoutException(String.format("The file %s caused a timeout",fileId), e, false, RuleFileType.ENTITY); } throw new RuntimeException(e); } catch (InterruptedException e) { kieSession.dispose(); throw new RuntimeException(e); } catch (TimeoutException e) { - throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY); + throw new DroolsTimeoutException(String.format("The file %s caused a timeout",fileId), e, false, RuleFileType.ENTITY); } List resultingFileAttributes = getFileAttributes(kieSession); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java index e4ee9642..e4723961 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java @@ -22,6 +22,15 @@ public class DroolsTimeoutException extends RuntimeException { } + public DroolsTimeoutException(String message, Throwable cause, boolean reported, RuleFileType ruleFileType) { + + super(message, cause); + this.reported = reported; + this.ruleFileType = ruleFileType; + } + + + public DroolsTimeoutException(boolean reported, RuleFileType ruleFileType) { super(DROOLS_TIMEOUT_MESSAGE); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 8692f648..8085059c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -154,7 +154,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart); long droolsStart = System.currentTimeMillis(); - List fileAttributes = entityDroolsExecutionService.executeRules(kieContainer, + List fileAttributes = entityDroolsExecutionService.executeRules(TEST_FILE_ID, kieContainer, document, dictionary, Collections.emptyList(), diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta index 5705cc07..57e6e0dd 160000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta @@ -1 +1 @@ -Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421 +Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7