From 8961abc4febeb019dd9e0980e4399f65c00d0ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Wed, 24 Apr 2024 11:50:13 +0200 Subject: [PATCH] Resolve RED-7384 --- .../redaction/v1/server/RedactionServiceSettings.java | 6 ++++++ .../redaction/v1/server/queue/MessageReceiver.java | 2 +- .../v1/server/queue/PriorityMessageReceiver.java | 2 +- .../drools/ComponentDroolsExecutionService.java | 11 ++++++++--- .../service/drools/EntityDroolsExecutionService.java | 2 +- .../redaction/v1/server/AnalysisEnd2EndTest.java | 2 +- .../src/test/resources/application.yml | 2 ++ 7 files changed, 20 insertions(+), 7 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index 2cc7f8f4..de520afa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -30,4 +30,10 @@ public class RedactionServiceSettings { private int droolsExecutionTimeoutSecs = 300; + + public int getDroolsExecutionTimeoutSecs(int numberOfPages) { + + return (int) Math.max(getDroolsExecutionTimeoutSecs(), getDroolsExecutionTimeoutSecs() * ((float) numberOfPages / 1000)); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java index fdd9c96d..c3766d19 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java @@ -17,7 +17,7 @@ public class MessageReceiver { @RabbitHandler - @RabbitListener(queues = REDACTION_QUEUE) + @RabbitListener(queues = REDACTION_QUEUE, concurrency = "1") public void receiveAnalyzeRequest(Message message) { redactionMessageReceiver.receiveAnalyzeRequest(message, false); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java index 9e099b3d..93f86ee8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java @@ -17,7 +17,7 @@ public class PriorityMessageReceiver { @RabbitHandler - @RabbitListener(queues = REDACTION_PRIORITY_QUEUE) + @RabbitListener(queues = REDACTION_PRIORITY_QUEUE, concurrency = "1") public void receiveAnalyzeRequest(Message message) { redactionMessageReceiver.receiveAnalyzeRequest(message, true); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java index 4ff0f3f2..395d4b66 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java @@ -50,7 +50,9 @@ public class ComponentDroolsExecutionService { .filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED)) .map(entry -> Entity.fromEntityLogEntry(entry, document)) .forEach(kieSession::insert); - fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert); + fileAttributes.stream() + .filter(f -> f.getValue() != null) + .forEach(kieSession::insert); CompletableFuture completableFuture = CompletableFuture.supplyAsync(() -> { kieSession.fireAllRules(); @@ -58,7 +60,8 @@ public class ComponentDroolsExecutionService { }); try { - completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get(); + completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS) + .get(); } catch (ExecutionException e) { kieSession.dispose(); if (e.getCause() instanceof TimeoutException) { @@ -71,7 +74,9 @@ public class ComponentDroolsExecutionService { } List resultingFileAttributes = getFileAttributes(kieSession); - List components = getComponents(kieSession).stream().sorted(ComponentComparator.first()).toList(); + List components = getComponents(kieSession).stream() + .sorted(ComponentComparator.first()) + .toList(); kieSession.dispose(); return components; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index 9fb33529..de201f33 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -122,7 +122,7 @@ public class EntityDroolsExecutionService { }); try { - completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS) + completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS) .get(); } catch (ExecutionException e) { kieSession.dispose(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index e79035e5..18c3a2ce 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -104,7 +104,7 @@ import lombok.extern.slf4j.Slf4j; @SneakyThrows public void runAnalysisEnd2End() { - String folder = "files/end2end/file0"; // Should contain all files from minio directly, still zipped. Can contain multiple files. + String folder = "files/end2end/file0"; // Should contain all files from minio directly, still zipped. Can contain multiple fileIds. Path absoluteFolderPath; if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml index 2d87538c..638f76bc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml @@ -25,6 +25,8 @@ redaction-service: application: type: "RedactManager" +logging.type: "CONSOLE" + storage: backend: 's3'