From d1c2d43ffb9f9c001ebd17494d444ac5eff97ce1 Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Tue, 27 Aug 2024 13:22:18 +0200 Subject: [PATCH] RED-9331: Explore possibilities for fair upload / analysis processing per tenant --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../redaction/v1/model/QueueNames.java | 11 +-- .../build.gradle.kts | 11 +-- ...orityTenantMessagingConfigurationImpl.java | 12 ++++ .../TenantMessagingConfigurationImpl.java | 12 ++++ .../v1/server/queue/MessageReceiver.java | 5 +- .../server/queue/MessagingConfiguration.java | 40 +---------- .../server/queue/PriorityMessageReceiver.java | 6 +- ...rityTenantExchangeMessageReceiverImpl.java | 68 +++++++++++++++++++ .../queue/RedactionMessageReceiver.java | 15 ++-- .../TenantExchangeMessageReceiverImpl.java | 68 +++++++++++++++++++ .../service/UnprocessedChangesService.java | 4 +- .../storage/RedactionStorageService.java | 14 +++- .../AbstractRedactionIntegrationTest.java | 8 +++ .../v1/server/DictionaryServiceTest.java | 8 +++ .../redaction/v1/server/RulesTest.java | 6 ++ .../v1/server/document/graph/TableTest.java | 2 - .../realdata/LiveDataIntegrationTest.java | 8 +++ .../UnprocessedChangesServiceTest.java | 12 ++-- .../utils/EntityVisualizationUtility.java | 2 +- .../src/test/resources/application.yml | 3 + 21 files changed, 248 insertions(+), 69 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/PriorityTenantMessagingConfigurationImpl.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/TenantMessagingConfigurationImpl.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityTenantExchangeMessageReceiverImpl.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/TenantExchangeMessageReceiverImpl.java diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index abc085cd..78bc2b3f 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.531.0" +val persistenceServiceVersion = "2.532.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java index ee95ce60..ba2a1849 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java @@ -2,10 +2,13 @@ package com.iqser.red.service.redaction.v1.model; public class QueueNames { - public static final String REDACTION_QUEUE = "redactionQueue"; - public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue"; - public static final String REDACTION_ANALYSIS_RESPONSE_QUEUE = "redactionAnalysisResponseQueue"; - public static final String REDACTION_DQL = "redactionDQL"; + public static final String REDACTION_REQUEST_QUEUE_PREFIX = "redaction_request_queue"; + public static final String REDACTION_REQUEST_EXCHANGE = "redaction_request_exchange"; + public static final String REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX = "redaction_request_queue"; + public static final String REDACTION_PRIORITY_REQUEST_EXCHANGE = "redaction_priority_request_exchange"; + public static final String REDACTION_RESPONSE_EXCHANGE = "redaction_response_exchange"; + public static final String REDACTION_DLQ = "redaction_dlq"; + public static final String MIGRATION_QUEUE = "migrationQueue"; public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue"; diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 2e900250..a5a6e1a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -12,11 +12,11 @@ plugins { description = "redaction-service-server-v1" -val layoutParserVersion = "0.160.0" +val layoutParserVersion = "0.161.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.531.0" +val persistenceServiceVersion = "2.532.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" @@ -35,6 +35,9 @@ dependencies { implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") + { + exclude(group = "com.knecon.fforesight", module = "tenant-commons") + } implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.iqser.red.commons:spring-commons:6.2.0") @@ -42,8 +45,8 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.45.0") - implementation("com.knecon.fforesight:keycloak-commons:0.29.0") - implementation("com.knecon.fforesight:tenant-commons:0.25.0") + implementation("com.knecon.fforesight:tenant-commons:0.28.0") + implementation("com.knecon.fforesight:keycloak-commons:0.30.0") implementation("com.knecon.fforesight:tracing-commons:0.5.0") implementation("com.knecon.fforesight:lifecycle-commons:0.6.0") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/PriorityTenantMessagingConfigurationImpl.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/PriorityTenantMessagingConfigurationImpl.java new file mode 100644 index 00000000..2ec3a694 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/PriorityTenantMessagingConfigurationImpl.java @@ -0,0 +1,12 @@ +package com.iqser.red.service.redaction.v1.server.configuraton; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; + +import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration; + +@Configuration +@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true") +public class PriorityTenantMessagingConfigurationImpl extends TenantMessagingConfiguration { + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/TenantMessagingConfigurationImpl.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/TenantMessagingConfigurationImpl.java new file mode 100644 index 00000000..9d54f3be --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/TenantMessagingConfigurationImpl.java @@ -0,0 +1,12 @@ +package com.iqser.red.service.redaction.v1.server.configuraton; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; + +import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration; + +@Configuration +@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false") +public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration { + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java index c3766d19..782275eb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.queue; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE; import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.annotation.RabbitHandler; @@ -13,11 +12,13 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class MessageReceiver { + public static final String REDACTION_REQUEST_LISTENER_ID = "redaction-request-listener"; + private final RedactionMessageReceiver redactionMessageReceiver; @RabbitHandler - @RabbitListener(queues = REDACTION_QUEUE, concurrency = "1") + @RabbitListener(id = REDACTION_REQUEST_LISTENER_ID, concurrency = "1") public void receiveAnalyzeRequest(Message message) { redactionMessageReceiver.receiveAnalyzeRequest(message, false); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java index 1c5a3f9d..430b8d63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java @@ -1,11 +1,6 @@ package com.iqser.red.service.redaction.v1.server.queue; -import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ; -import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE; +import static com.iqser.red.service.redaction.v1.model.QueueNames.*; import org.springframework.amqp.core.Queue; import org.springframework.amqp.core.QueueBuilder; @@ -54,38 +49,9 @@ public class MessagingConfiguration { @Bean - public Queue redactionQueue() { + public Queue redactionDLQ() { - return QueueBuilder.durable(REDACTION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REDACTION_DQL).maxPriority(2).build(); - } - - - @Bean - public Queue redactionPriorityQueue() { - - return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", REDACTION_DQL) - .maxPriority(2) - .build(); - } - - - @Bean - public Queue redactionAnalysisResponseQueue() { - - return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", REDACTION_DQL) - .maxPriority(2) - .build(); - } - - - @Bean - public Queue redactionDeadLetterQueue() { - - return QueueBuilder.durable(REDACTION_DQL).build(); + return QueueBuilder.durable(REDACTION_DLQ).build(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java index 93f86ee8..d978c2ea 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java @@ -1,7 +1,5 @@ package com.iqser.red.service.redaction.v1.server.queue; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE; - import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.annotation.RabbitHandler; import org.springframework.amqp.rabbit.annotation.RabbitListener; @@ -13,11 +11,13 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class PriorityMessageReceiver { + public static final String REDACTION_PRIORITY_REQUEST_LISTENER_ID = "redaction-priority-request-listener"; + private final RedactionMessageReceiver redactionMessageReceiver; @RabbitHandler - @RabbitListener(queues = REDACTION_PRIORITY_QUEUE, concurrency = "1") + @RabbitListener(id = REDACTION_PRIORITY_REQUEST_LISTENER_ID, concurrency = "1") public void receiveAnalyzeRequest(Message message) { redactionMessageReceiver.receiveAnalyzeRequest(message, true); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityTenantExchangeMessageReceiverImpl.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityTenantExchangeMessageReceiverImpl.java new file mode 100644 index 00000000..9e14ec4c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityTenantExchangeMessageReceiverImpl.java @@ -0,0 +1,68 @@ +package com.iqser.red.service.redaction.v1.server.queue; + +import static com.iqser.red.service.redaction.v1.model.QueueNames.*; +import java.util.Map; +import java.util.Set; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import com.knecon.fforesight.tenantcommons.TenantProvider; +import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent; +import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration; +import com.knecon.fforesight.tenantcommons.model.TenantResponse; +import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService; +import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver; + +@Service +@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true") +public class PriorityTenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver { + + public PriorityTenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) { + + super(rabbitQueueService, tenantProvider); + } + + + @Override + protected Set getTenantQueueConfigs() { + + return Set.of(TenantQueueConfiguration.builder() + .listenerId(PriorityMessageReceiver.REDACTION_PRIORITY_REQUEST_LISTENER_ID) + .exchangeName(REDACTION_PRIORITY_REQUEST_EXCHANGE) + .queuePrefix(REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX) + .dlqName(REDACTION_DLQ) + .arguments(Map.of("x-max-priority", 2)) + .build()); + } + + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationReady() { + + System.out.println("application ready invoked"); + super.initializeQueues(); + } + + + @RabbitHandler + @RabbitListener(queues = "#{priorityTenantMessagingConfigurationImpl.getTenantCreatedQueueName()}") + public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) { + + super.reactToTenantCreation(tenantCreatedEvent); + } + + + @RabbitHandler + @RabbitListener(queues = "#{priorityTenantMessagingConfigurationImpl.getTenantDeletedQueueName()}") + public void reactToTenantDeletion(TenantResponse tenantResponse) { + + super.reactToTenantDeletion(tenantResponse); + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index 851f7dac..a75aa2ec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -1,8 +1,6 @@ package com.iqser.red.service.redaction.v1.server.queue; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE; -import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE; +import static com.iqser.red.service.redaction.v1.model.QueueNames.*; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER; import static java.lang.String.format; @@ -56,7 +54,7 @@ public class RedactionMessageReceiver { fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(errorMessage, - priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, + priority ? REDACTION_PRIORITY_REQUEST_EXCHANGE : REDACTION_REQUEST_EXCHANGE, "redaction-service", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); throw new AmqpRejectAndDontRequeueException(errorMessage); @@ -143,12 +141,15 @@ public class RedactionMessageReceiver { var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), - new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); + new FileErrorInfo(e.getMessage(), + priority ? REDACTION_PRIORITY_REQUEST_EXCHANGE : REDACTION_REQUEST_EXCHANGE, + "redaction-service", + timestamp)); } @RabbitHandler - @RabbitListener(queues = REDACTION_DQL) + @RabbitListener(queues = REDACTION_DLQ) public void receiveAnalyzeRequestDLQ(Message in) throws IOException { var analyzeRequest = objectMapper.readValue(in.getBody(), AnalyzeRequest.class); @@ -159,7 +160,7 @@ public class RedactionMessageReceiver { log.info("Failed to process analyze request, errorCause: {}, timestamp: {}", errorCause, timestamp); fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), - new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp)); + new FileErrorInfo(errorCause, REDACTION_DLQ, "redaction-service", timestamp)); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/TenantExchangeMessageReceiverImpl.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/TenantExchangeMessageReceiverImpl.java new file mode 100644 index 00000000..4bae9d80 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/TenantExchangeMessageReceiverImpl.java @@ -0,0 +1,68 @@ +package com.iqser.red.service.redaction.v1.server.queue; + +import static com.iqser.red.service.redaction.v1.model.QueueNames.*; +import java.util.Map; +import java.util.Set; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import com.knecon.fforesight.tenantcommons.TenantProvider; +import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent; +import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration; +import com.knecon.fforesight.tenantcommons.model.TenantResponse; +import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService; +import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver; + +@Service +@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false") +public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver { + + public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) { + + super(rabbitQueueService, tenantProvider); + } + + + @Override + protected Set getTenantQueueConfigs() { + + return Set.of(TenantQueueConfiguration.builder() + .listenerId(MessageReceiver.REDACTION_REQUEST_LISTENER_ID) + .exchangeName(REDACTION_REQUEST_EXCHANGE) + .queuePrefix(REDACTION_REQUEST_QUEUE_PREFIX) + .dlqName(REDACTION_DLQ) + .arguments(Map.of("x-max-priority", 2)) + .build()); + } + + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationReady() { + + System.out.println("application ready invoked"); + super.initializeQueues(); + } + + + @RabbitHandler + @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}") + public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) { + + super.reactToTenantCreation(tenantCreatedEvent); + } + + + @RabbitHandler + @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}") + public void reactToTenantDeletion(TenantResponse tenantResponse) { + + super.reactToTenantDeletion(tenantResponse); + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java index 0e3e1ef9..8a72f897 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java @@ -31,6 +31,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingU import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; @@ -127,7 +128,8 @@ public class UnprocessedChangesService { notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity))); - rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE, + rabbitTemplate.convertAndSend(QueueNames.REDACTION_RESPONSE_EXCHANGE, + TenantContext.getTenantId(), AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index e76417f3..742c4d0a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server.storage; +import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration.APPLICATION_TASK_EXECUTOR_BEAN_NAME; + import java.io.File; import java.io.FileInputStream; import java.io.InputStream; @@ -10,6 +12,7 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import java.util.stream.Collectors; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.cache.annotation.Cacheable; import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Service; @@ -42,7 +45,6 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service -@RequiredArgsConstructor public class RedactionStorageService { private final StorageService storageService; @@ -52,6 +54,16 @@ public class RedactionStorageService { private final TaskExecutor taskExecutor; + public RedactionStorageService(StorageService storageService, + EntityLogMongoService entityLogMongoService, + @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) { + + this.storageService = storageService; + this.entityLogMongoService = entityLogMongoService; + this.taskExecutor = taskExecutor; + } + + @SneakyThrows public InputStream getStoredObject(String storageId) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index c887000b..96f39cd6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -27,7 +27,9 @@ import org.bson.BsonString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.mockito.stubbing.Answer; +import org.springframework.amqp.rabbit.core.RabbitAdmin; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.util.TestPropertyValues; @@ -162,6 +164,12 @@ public abstract class AbstractRedactionIntegrationTest { @MockBean protected RabbitTemplate rabbitTemplate; + @MockBean + protected RabbitAdmin rabbitAdmin; + + @MockBean + protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry; + @MockBean protected LegalBasisClient legalBasisClient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index 6738388c..b92b6ae2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -15,7 +15,9 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.kie.api.runtime.KieContainer; +import org.springframework.amqp.rabbit.core.RabbitAdmin; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; @@ -52,6 +54,12 @@ public class DictionaryServiceTest { @MockBean private RabbitTemplate rabbitTemplate; + @MockBean + protected RabbitAdmin rabbitAdmin; + + @MockBean + protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry; + @MockBean private TenantsClient tenantsClient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index ea5fc214..8390e789 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -40,7 +40,9 @@ import org.kie.api.builder.KieBuilder; import org.kie.api.builder.KieFileSystem; import org.kie.api.builder.KieModule; import org.kie.api.runtime.KieContainer; +import org.springframework.amqp.rabbit.core.RabbitAdmin; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; @@ -254,6 +256,10 @@ public class RulesTest { @MockBean private RabbitTemplate rabbitTemplate; @MockBean + protected RabbitAdmin rabbitAdmin; + @MockBean + protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry; + @MockBean private LegalBasisClient legalBasisClient; @MockBean private TenantsClient tenantsClient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java index c53a5eb3..e2e33390 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.document.graph; -import static com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility.ENTITY_LAYER; import static org.junit.jupiter.api.Assertions.assertEquals; import java.awt.Color; @@ -25,7 +24,6 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityCreation import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility; -import com.knecon.fforesight.service.viewerdoc.model.Visualizations; import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService; import com.knecon.fforesight.tenantcommons.TenantContext; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java index 3f845ec5..f2cfea78 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java @@ -18,7 +18,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.amqp.rabbit.core.RabbitAdmin; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; @@ -83,6 +85,12 @@ public class LiveDataIntegrationTest { @MockBean private RabbitTemplate rabbitTemplate; + @MockBean + protected RabbitAdmin rabbitAdmin; + + @MockBean + protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry; + @MockBean private TenantsClient tenantsClient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index 7558c2e8..c00a9e2d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -150,7 +150,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT request.setManualRedactions(manualRedactions); unprocessedChangesService.analyseSurroundingText(request); - verify(rabbitTemplate).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + verify(rabbitTemplate).convertAndSend(eq(QueueNames.REDACTION_RESPONSE_EXCHANGE), eq(TenantContext.getTenantId()), captor.capture()); List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); @@ -203,7 +203,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); unprocessedChangesService.analyseSurroundingText(request); - verify(rabbitTemplate, times(2)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + verify(rabbitTemplate, times(2)).convertAndSend(eq(QueueNames.REDACTION_RESPONSE_EXCHANGE), eq(TenantContext.getTenantId()), captor.capture()); unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); @@ -282,7 +282,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction, manualResizeRedaction2, manualResizeRedaction3)).build()); unprocessedChangesService.analyseSurroundingText(request); - verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_RESPONSE_EXCHANGE), eq(TenantContext.getTenantId()), captor.capture()); List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); @@ -368,7 +368,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); unprocessedChangesService.analyseSurroundingText(request); - verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_RESPONSE_EXCHANGE), eq(TenantContext.getTenantId()), captor.capture()); List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); @@ -418,7 +418,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); unprocessedChangesService.analyseSurroundingText(request); - verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_RESPONSE_EXCHANGE), eq(TenantContext.getTenantId()), captor.capture()); List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); @@ -468,7 +468,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); unprocessedChangesService.analyseSurroundingText(request); - verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_RESPONSE_EXCHANGE), eq(TenantContext.getTenantId()), captor.capture()); List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/EntityVisualizationUtility.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/EntityVisualizationUtility.java index abe5d49b..a0ef7b5c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/EntityVisualizationUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/EntityVisualizationUtility.java @@ -24,7 +24,7 @@ public class EntityVisualizationUtility { public static final LayerIdentifier ENTITY_LAYER = new LayerIdentifier("Entities", "KNECON_ENTITIES"); - public Map createVisualizationsOnPage(Collection entity, Color color) { + private Map createVisualizationsOnPage(Collection entity, Color color) { Map visualizations = new HashMap<>(); Set pages = entity.stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml index d3fd3a11..1e6d3649 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml @@ -53,3 +53,6 @@ multitenancy: mongo: liquibase: changeLog: classpath:mongo/changelog/mongo.changelog-tenant.xml + + +POD_NAME: redaction-service