Compare commits

..

2 Commits

Author SHA1 Message Date
Maverick Studer
318b5f49fe Merge branch 'update-psv' into 'release/2.92.x'
Update persistence-service version

See merge request redactmanager/search-service!31
2024-09-30 11:48:01 +02:00
maverickstuder
ed93c8c6fb Update persistence-service version 2024-09-30 11:33:29 +02:00
11 changed files with 89 additions and 191 deletions

View File

@ -22,13 +22,13 @@ configurations {
val springBootStarterVersion = "3.1.5"
dependencies {
api("com.knecon.fforesight:tenant-commons:0.30.0")
api("com.knecon.fforesight:tenant-commons:0.21.0")
api("com.knecon.fforesight:tracing-commons:0.5.0")
api("com.knecon.fforesight:lifecycle-commons:0.6.0")
api("com.google.guava:guava:31.1-jre")
api("com.iqser.red.commons:storage-commons:2.45.0")
api(project(":search-service-api-v1"))
api("com.iqser.red.service:persistence-service-internal-api-v1:2.576.0-RED10106.0")
api("com.iqser.red.service:persistence-service-internal-api-v1:2.465.78")
api("com.iqser.red.commons:spring-commons:2.1.0")
api("com.iqser.red.commons:metric-commons:2.1.0")
api("com.iqser.red.commons:jackson-commons:2.1.0")

View File

@ -1,11 +0,0 @@
package com.iqser.red.service.search.v1.server.configuration;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
@Configuration
public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
}

View File

@ -1,12 +1,14 @@
package com.iqser.red.service.search.v1.server.queue;
import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.DELETE_FROM_INDEX_DLQ;
import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.INDEXING_DLQ;
import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.INDEXING_REQUEST_EXCHANGE;
import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_DLQ;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_QUEUE;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_DQL;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_QUEUE;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
@ -18,6 +20,7 @@ import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
@ -35,7 +38,6 @@ import com.iqser.red.service.search.v1.server.service.IndexDeleteService;
import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService;
import com.iqser.red.service.search.v1.server.service.IndexInformationService;
import com.iqser.red.service.search.v1.server.service.TextStorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@ -46,9 +48,6 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
public class IndexingMessageReceiver {
public static final String INDEXING_LISTENER_ID = "indexing-listener";
public static final String DELETE_FROM_INDEX_LISTENER_ID = "delete-from-index-listener";
private final ObjectMapper objectMapper;
private final TextStorageService textStorageService;
private final FileStatusClient fileStatusClient;
@ -66,7 +65,7 @@ public class IndexingMessageReceiver {
@SneakyThrows
@RabbitHandler
@RabbitListener(id = INDEXING_LISTENER_ID)
@RabbitListener(queues = INDEXING_QUEUE)
public void receiveIndexingRequest(Message message) {
var indexRequest = objectMapper.readValue(message.getBody(), IndexMessage.class);
@ -74,8 +73,8 @@ public class IndexingMessageReceiver {
// This prevents from endless retries oom errors.
if (message.getMessageProperties().isRedelivered()) {
throw new AmqpRejectAndDontRequeueException(String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
indexRequest.getDossierId(),
indexRequest.getFileId()));
indexRequest.getDossierId(),
indexRequest.getFileId()));
}
try {
@ -106,11 +105,12 @@ public class IndexingMessageReceiver {
fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true);
if(documentUpdateService.documentExists(indexRequest.getFileId())) {var indexUpdateDocument = indexDocumentConverterService.convertUpdateDocument(fileStatus.getAssignee(),
dossier.getSoftDeletedTime() != null,
dossier.getArchivedTime() != null,
fileStatus.getWorkflowStatus().name(),
fileStatus.getFileAttributes());
if(documentUpdateService.documentExists(indexRequest.getFileId())) {
var indexUpdateDocument = indexDocumentConverterService.convertUpdateDocument(fileStatus.getAssignee(),
dossier.getSoftDeletedTime() != null,
dossier.getArchivedTime() != null,
fileStatus.getWorkflowStatus().name(),
fileStatus.getFileAttributes());
documentUpdateService.updateDocument(indexRequest.getFileId(), indexUpdateDocument);
log.info("Successfully updated {}", indexRequest);
@ -133,7 +133,7 @@ public class IndexingMessageReceiver {
@RabbitHandler
@RabbitListener(queues = INDEXING_DLQ)
@RabbitListener(queues = INDEXING_DQL)
public void receiveIndexingRequestDQL(Message in) throws IOException {
var indexRequest = objectMapper.readValue(in.getBody(), IndexMessage.class);
@ -142,15 +142,13 @@ public class IndexingMessageReceiver {
String errorMessage = errorLog + in.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
OffsetDateTime timestamp = in.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(),
indexRequest.getFileId(),
new FileErrorInfo(errorMessage, INDEXING_DLQ, "search-service", timestamp));
fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(), indexRequest.getFileId(), new FileErrorInfo(errorMessage, INDEXING_DQL, "search-service", timestamp));
}
@RabbitHandler
@RabbitListener(id = DELETE_FROM_INDEX_LISTENER_ID)
@RabbitListener(queues = DELETE_FROM_INDEX_QUEUE)
public void receiveDeleteDocumentRequest(Message in) throws IOException {
var indexRequest = objectMapper.readValue(in.getBody(), IndexMessage.class);
@ -179,9 +177,7 @@ public class IndexingMessageReceiver {
String errorMessage = errorLog + in.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
OffsetDateTime timestamp = in.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(),
indexRequest.getFileId(),
new FileErrorInfo(errorMessage, INDEXING_DLQ, "search-service", timestamp));
fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(), indexRequest.getFileId(), new FileErrorInfo(errorMessage, INDEXING_DQL, "search-service", timestamp));
}
@ -192,15 +188,15 @@ public class IndexingMessageReceiver {
Text text = textStorageService.getText(dossier.getId(), file.getId());
var indexDocument = indexDocumentConverterService.convert(dossier.getDossierTemplateId(),
dossier.getId(),
file.getId(),
file.getFilename(),
text,
file.getAssignee(),
dossier.getSoftDeletedTime() != null,
dossier.getArchivedTime() != null,
file.getWorkflowStatus(),
file.getFileAttributes());
dossier.getId(),
file.getId(),
file.getFilename(),
text,
file.getAssignee(),
dossier.getSoftDeletedTime() != null,
dossier.getArchivedTime() != null,
file.getWorkflowStatus(),
file.getFileAttributes());
documentIndexService.indexDocument(indexDocument);
fileStatusProcessingUpdateClient.indexingSuccessful(dossier.getId(), file.getId());
@ -223,13 +219,12 @@ public class IndexingMessageReceiver {
for (FileModel file : files) {
log.info("Will add dossier {} file {} to index queue", dossierId, file.getId());
rabbitTemplate.convertAndSend(INDEXING_REQUEST_EXCHANGE,
TenantContext.getTenantId(),
IndexMessage.builder().messageType(IndexMessageType.INSERT).dossierId(dossierId).fileId(file.getId()).build(),
message -> {
message.getMessageProperties().setPriority(99);
return message;
});
rabbitTemplate.convertAndSend(INDEXING_QUEUE,
IndexMessage.builder().messageType(IndexMessageType.INSERT).dossierId(dossierId).fileId(file.getId()).build(),
message -> {
message.getMessageProperties().setPriority(99);
return message;
});
}
}

View File

@ -1,8 +1,7 @@
package com.iqser.red.service.search.v1.server.configuration;
package com.iqser.red.service.search.v1.server.queue;
import org.springframework.amqp.core.Binding;
import org.springframework.amqp.core.BindingBuilder;
import org.springframework.amqp.core.DirectExchange;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
import org.springframework.amqp.core.TopicExchange;
@ -17,46 +16,64 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class MessagingConfiguration {
public static final String INDEXING_REQUEST_QUEUE_PREFIX = "indexing_request";
public static final String INDEXING_REQUEST_EXCHANGE = "indexing_request_exchange";
public static final String INDEXING_DLQ = "indexing_error";
public static final String INDEXING_QUEUE = "indexingQueue";
public static final String INDEXING_DQL = "indexingDQL";
public static final String DELETE_FROM_INDEX_REQUEST_QUEUE_PREFIX = "delete_from_index_request";
public static final String DELETE_FROM_INDEX_REQUEST_EXCHANGE = "delete_from_index_request_exchange";
public static final String DELETE_FROM_INDEX_DLQ = "delete_from_index_error";
public static final String DELETE_FROM_INDEX_QUEUE = "deleteFromIndexQueue";
public static final String DELETE_FROM_INDEX_DLQ = "deleteFromIndexDLQ";
public static final String X_ERROR_INFO_HEADER = "x-error-message";
public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
@Value("${fforesight.multitenancy.tenant-delete-queue:search-service-tenant-delete}")
@Value("${fforesight.multitenancy.tenant-delete-queue:tenant-delete-queue}")
private String tenantDeleteEventQueueName;
@Value("${fforesight.multitenancy.tenant-delete-dlq:search-service-tenant-delete-error}")
@Value("${fforesight.multitenancy.tenant-delete-dlq:tenant-delete-dlq}")
private String tenantDeleteDLQName;
@Value("${fforesight.multitenancy.tenant-updated-queue:search-service-tenant-updated}")
@Value("${fforesight.multitenancy.tenant-updated-queue:tenant-updated-queue}")
private String tenantUpdatedEventQueueName;
@Value("${fforesight.multitenancy.tenant-updated-dlq:search-service-tenant-updated-error}")
@Value("${fforesight.multitenancy.tenant-updated-dlq:tenant-updated-dlq}")
private String tenantUpdatedDLQName;
@Bean
public DirectExchange indexingRequestExchange() {
public Binding tenantExchangeDeleteBinding(@Qualifier("tenantUserManagementTenantDeleteQueue") Queue tenantUserManagementTenantDeleteQueue,
@Qualifier("tenantExchange") TopicExchange tenantExchange) {
return new DirectExchange(INDEXING_REQUEST_EXCHANGE);
return BindingBuilder.bind(tenantUserManagementTenantDeleteQueue).to(tenantExchange).with("tenant.delete");
}
@Bean
public Queue indexingDLQ() {
public Binding tenantExchangeUpdatedBinding(@Qualifier("tenantUserManagementTenantUpdatedQueue") Queue tenantUserManagementTenantUpdatedQueue,
@Qualifier("tenantExchange") TopicExchange tenantExchange) {
return QueueBuilder.durable(INDEXING_DLQ).build();
return BindingBuilder.bind(tenantUserManagementTenantUpdatedQueue).to(tenantExchange).with("tenant.updated");
}
@Bean
public DirectExchange deleteFromIndexRequestExchange() {
public Queue indexingQueue() {
return new DirectExchange(DELETE_FROM_INDEX_REQUEST_EXCHANGE);
return QueueBuilder.durable(INDEXING_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", INDEXING_DQL).maxPriority(2).build();
}
@Bean
public Queue indexingDeadLetterQueue() {
return QueueBuilder.durable(INDEXING_DQL).build();
}
@Bean
public Queue deleteFromIndexQueue() {
return QueueBuilder.durable(DELETE_FROM_INDEX_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", DELETE_FROM_INDEX_DLQ)
.maxPriority(2)
.build();
}
@ -66,12 +83,13 @@ public class MessagingConfiguration {
return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build();
}
// Tentant Delete Event Queue
@Bean
public Binding tenantExchangeDeleteBinding(@Qualifier("tenantUserManagementTenantDeleteQueue") Queue tenantUserManagementTenantDeleteQueue,
@Qualifier("tenantExchange") TopicExchange tenantExchange) {
return BindingBuilder.bind(tenantUserManagementTenantDeleteQueue).to(tenantExchange).with("tenant.delete");
@Bean(name = "tenantExchange")
TopicExchange tenantExchange(@Value("${fforesight.tenant-exchange.name}") String tenantExchangeName) {
return new TopicExchange(tenantExchangeName);
}
@ -92,14 +110,6 @@ public class MessagingConfiguration {
}
@Bean
public Binding tenantExchangeUpdatedBinding(@Qualifier("tenantUserManagementTenantUpdatedQueue") Queue tenantUserManagementTenantUpdatedQueue,
@Qualifier("tenantExchange") TopicExchange tenantExchange) {
return BindingBuilder.bind(tenantUserManagementTenantUpdatedQueue).to(tenantExchange).with("tenant.updated");
}
@Bean("tenantUserManagementTenantUpdatedQueue")
public Queue tenantUpdatedQueue() {

View File

@ -1,74 +0,0 @@
package com.iqser.red.service.search.v1.server.queue;
import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.*;
import java.util.Map;
import java.util.Set;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
@Service
public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
super(rabbitQueueService, tenantProvider);
}
@Override
protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
return Set.of(TenantQueueConfiguration.builder()
.listenerId(IndexingMessageReceiver.INDEXING_LISTENER_ID)
.exchangeName(INDEXING_REQUEST_EXCHANGE)
.queuePrefix(INDEXING_REQUEST_QUEUE_PREFIX)
.dlqName(INDEXING_DLQ)
.arguments(Map.of("x-max-priority", 2))
.build(),
TenantQueueConfiguration.builder()
.listenerId(IndexingMessageReceiver.DELETE_FROM_INDEX_LISTENER_ID)
.exchangeName(DELETE_FROM_INDEX_REQUEST_EXCHANGE)
.queuePrefix(DELETE_FROM_INDEX_REQUEST_QUEUE_PREFIX)
.dlqName(DELETE_FROM_INDEX_DLQ)
.arguments(Map.of("x-max-priority", 2))
.build());
}
@EventListener(ApplicationReadyEvent.class)
public void onApplicationReady() {
System.out.println("application ready invoked");
super.initializeQueues();
}
@RabbitHandler
@RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
super.reactToTenantCreation(tenantCreatedEvent);
}
@RabbitHandler
@RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
public void reactToTenantDeletion(TenantResponse tenantResponse) {
super.reactToTenantDeletion(tenantResponse);
}
}

View File

@ -21,7 +21,7 @@ public class UpdatedTenantMessageReceiver {
private final IndexQueryService indexQueryService;
private final IndexDeleteService indexDeleteService;
@Value("${fforesight.multitenancy.tenant-updated-queue:search-service-tenant-updated}")
@Value("${fforesight.multitenancy.tenant-updated-queue:tenant-updated-queue}")
private String tenantUpdatedQueue;
@ -32,7 +32,7 @@ public class UpdatedTenantMessageReceiver {
}
@RabbitListener(queues = "${fforesight.multitenancy.tenant-updated-queue:search-service-tenant-updated}")
@RabbitListener(queues = "${fforesight.multitenancy.tenant-updated-queue:tenant-updated-queue}")
public void updateTenant(TenantResponse tenant) {
String numberOfReplicas = tenant.getSearchConnection().getNumberOfReplicas();

View File

@ -6,8 +6,6 @@ import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@ -54,12 +52,6 @@ public abstract class AbstractElasticsearchIntegrationTest {
@MockBean
private TenantsClient tenantsClient;
@MockBean
private RabbitAdmin rabbitAdmin;
@MockBean
private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
private static int port;
@ -68,15 +60,15 @@ public abstract class AbstractElasticsearchIntegrationTest {
TenantContext.setTenantId("redaction");
when(tenantsClient.getTenant("redaction")).thenReturn(TenantResponse.builder()
.searchConnection(SearchConnection.builder()
.hosts(Set.of("localhost"))
.port(port)
.scheme("http")
.numberOfShards("1")
.numberOfReplicas("5")
.indexPrefix("indexprefix")
.build())
.build());
.searchConnection(SearchConnection.builder()
.hosts(Set.of("localhost"))
.port(port)
.scheme("http")
.numberOfShards("1")
.numberOfReplicas("5")
.indexPrefix("indexprefix")
.build())
.build());
}

View File

@ -7,9 +7,7 @@ import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;

View File

@ -3,9 +3,7 @@ package com.iqser.red.service.search.v1.server.service;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.Test;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;

View File

@ -7,9 +7,7 @@ import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
@ -56,12 +54,6 @@ public class OpensearchTest extends AbstractOpensearchIntegrationTest {
@MockBean
private RabbitTemplate rabbitTemplate;
@MockBean
private RabbitAdmin rabbitAdmin;
@MockBean
private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
@MockBean
private IndexDeleteService indexDeleteService;

View File

@ -32,5 +32,3 @@ persistence-service.url: 'http://mock.url'
server:
port: 19547
POD_NAME: search-service