RED-6686 Extract Tenant and user-management code into a separate service.

This commit is contained in:
Timo Bejan 2023-06-26 21:58:49 +02:00
parent eb91574b0a
commit 06a504cf2f
19 changed files with 50 additions and 419 deletions

View File

@ -12,13 +12,21 @@
<artifactId>ocr-service-server-v1</artifactId>
<properties>
<tennat-commons.version>0.6.0</tennat-commons.version>
</properties>
<dependencies>
<dependency>
<groupId>com.knecon.fforesight</groupId>
<artifactId>tenant-commons</artifactId>
<version>${tennat-commons.version}</version>
</dependency>
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>ocr-service-api-v1</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>storage-commons</artifactId>
@ -28,7 +36,6 @@
<artifactId>pdftron-logic-commons</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>spring-commons</artifactId>

View File

@ -2,6 +2,7 @@ package com.iqser.red.service.ocr.v1.server;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@ -13,19 +14,18 @@ import org.springframework.scheduling.annotation.EnableAsync;
import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.ocr.v1.server.multitenancy.AsyncConfig;
import com.iqser.red.service.ocr.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
import com.iqser.red.service.ocr.v1.server.multitenancy.MultiTenancyWebConfiguration;
import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
import io.micrometer.core.aop.TimedAspect;
import io.micrometer.core.instrument.MeterRegistry;
@EnableAsync
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@EnableConfigurationProperties(OcrServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class, StorageAutoConfiguration.class})
@Import({ MessagingConfiguration.class, StorageAutoConfiguration.class})
@EnableFeignClients(basePackageClasses = FileStatusProcessingUpdateClient.class)
public class Application {

View File

@ -1,10 +0,0 @@
package com.iqser.red.service.ocr.v1.server.client;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
public interface TenantsClient extends TenantsResource {
}

View File

@ -1,27 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import java.util.concurrent.Executor;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
public class AsyncConfig extends AsyncConfigurerSupport {
@Override
public Executor getAsyncExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(7);
executor.setMaxPoolSize(42);
executor.setQueueCapacity(11);
executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
executor.setTaskDecorator(new TenantAwareTaskDecorator());
executor.initialize();
return executor;
}
}

View File

@ -1,105 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import java.util.Base64;
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.SecretKeySpec;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import lombok.SneakyThrows;
@Service
public class EncryptionDecryptionService {
@Value("${ocr-service.crypto.key:redaction}")
private String key;
private SecretKey secretKey;
private byte[] iv;
@SneakyThrows
@PostConstruct
protected void postConstruct() {
SecureRandom secureRandom = new SecureRandom();
iv = new byte[12];
secureRandom.nextBytes(iv);
secretKey = generateSecretKey(key, iv);
}
@SneakyThrows
public String encrypt(String strToEncrypt) {
return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
}
@SneakyThrows
public String decrypt(String strToDecrypt) {
byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
return new String(decrypt(bytes), StandardCharsets.UTF_8);
}
@SneakyThrows
public byte[] encrypt(byte[] data) {
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
byte[] encryptedData = cipher.doFinal(data);
ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
byteBuffer.putInt(iv.length);
byteBuffer.put(iv);
byteBuffer.put(encryptedData);
return byteBuffer.array();
}
@SneakyThrows
public byte[] decrypt(byte[] encryptedData) {
ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
int noonceSize = byteBuffer.getInt();
if (noonceSize < 12 || noonceSize >= 16) {
throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
}
byte[] iv = new byte[noonceSize];
byteBuffer.get(iv);
SecretKey secretKey = generateSecretKey(key, iv);
byte[] cipherBytes = new byte[byteBuffer.remaining()];
byteBuffer.get(cipherBytes);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
return cipher.doFinal(cipherBytes);
}
@SneakyThrows
public SecretKey generateSecretKey(String password, byte[] iv) {
KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
return new SecretKeySpec(key, "AES");
}
}

View File

@ -1,17 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.stereotype.Component;
import feign.RequestInterceptor;
import feign.RequestTemplate;
@Component
public class ForwardTenantInterceptor implements RequestInterceptor {
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
@Override
public void apply(RequestTemplate template) {
template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
}
}

View File

@ -1,48 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import static com.iqser.red.service.ocr.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME;
import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class MultiTenancyMessagingConfiguration {
@Bean
public static BeanPostProcessor multitenancyBeanPostProcessor() {
return new BeanPostProcessor() {
@Override
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
if (bean instanceof RabbitTemplate) {
((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId());
return m;
});
} else if (bean instanceof AbstractRabbitListenerContainerFactory) {
((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME);
if (tenant != null) {
TenantContext.setTenantId(tenant);
} else {
throw new RuntimeException("No Tenant is set queue message");
}
return m;
});
}
return bean;
}
};
}
}

View File

@ -1,28 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
@Configuration
public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
private final TenantInterceptor tenantInterceptor;
@Autowired
public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
this.tenantInterceptor = tenantInterceptor;
}
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addWebRequestInterceptor(tenantInterceptor);
}
}

View File

@ -1,45 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.stereotype.Service;
import com.iqser.red.service.ocr.v1.server.client.TenantsClient;
import com.iqser.red.storage.commons.model.AzureStorageConnection;
import com.iqser.red.storage.commons.model.S3StorageConnection;
import com.iqser.red.storage.commons.service.StorageConnectionProvider;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class StorageConnectionProviderImpl implements StorageConnectionProvider {
private final TenantsClient tenantsClient;
private final EncryptionDecryptionService encryptionDecryptionService;
@Override
public AzureStorageConnection getAzureStorageConnection(String tenantId) {
var tenant = tenantsClient.getTenant(tenantId);
return AzureStorageConnection.builder()
.connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
.containerName(tenant.getAzureStorageConnection().getContainerName())
.build();
}
@Override
public S3StorageConnection getS3StorageConnection(String tenantId) {
var tenant = tenantsClient.getTenant(tenantId);
return S3StorageConnection.builder()
.key(tenant.getS3StorageConnection().getKey())
.secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
.signerType(tenant.getS3StorageConnection().getSignerType())
.bucketName(tenant.getS3StorageConnection().getBucketName())
.region(tenant.getS3StorageConnection().getRegion())
.endpoint(tenant.getS3StorageConnection().getEndpoint())
.build();
}
}

View File

@ -1,23 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.core.task.TaskDecorator;
import org.springframework.lang.NonNull;
public class TenantAwareTaskDecorator implements TaskDecorator {
@Override
@NonNull
public Runnable decorate(@NonNull Runnable runnable) {
String tenantId = TenantContext.getTenantId();
return () -> {
try {
TenantContext.setTenantId(tenantId);
runnable.run();
} finally {
TenantContext.setTenantId(null);
}
};
}
}

View File

@ -1,29 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public final class TenantContext {
private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
public static void setTenantId(String tenantId) {
log.debug("Setting tenantId to " + tenantId);
currentTenant.set(tenantId);
}
public static String getTenantId() {
return currentTenant.get();
}
public static void clear() {
currentTenant.remove();
}
}

View File

@ -1,35 +0,0 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.stereotype.Component;
import org.springframework.ui.ModelMap;
import org.springframework.web.context.request.WebRequest;
import org.springframework.web.context.request.WebRequestInterceptor;
@Component
public class TenantInterceptor implements WebRequestInterceptor {
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
@Override
public void preHandle(WebRequest request) {
if (request.getHeader(TENANT_HEADER_NAME) != null) {
TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
}
}
@Override
public void postHandle(WebRequest request, ModelMap model) {
TenantContext.clear();
}
@Override
public void afterCompletion(WebRequest request, Exception ex) {
}
}

View File

@ -7,9 +7,9 @@ import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.ocr.v1.server.model.image.ImageServiceResponse;
import com.iqser.red.service.ocr.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;

View File

@ -88,7 +88,7 @@ public class OCRService {
Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, true);
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).build()));
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).build());
// Optimization:
// When a page does not have a TextZone, PDFTron whites out the page. But, PDFTron scans it anyway, resulting in a longer runtime.
@ -114,11 +114,7 @@ public class OCRService {
++numProcessedPages;
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder()
.fileId(fileId)
.numberOfPagesToOCR(pageIdToRectCollection.size())
.numberOfOCRedPages(numProcessedPages)
.build()));
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).numberOfOCRedPages(numProcessedPages).build());
} catch (PDFNetException e) {
log.error("Failed to process page {}", pageId);
@ -130,12 +126,7 @@ public class OCRService {
Optimizer.optimize(pdfDoc);
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder()
.fileId(fileId)
.numberOfPagesToOCR(pageIdToRectCollection.size())
.numberOfOCRedPages(numProcessedPages)
.ocrFinished(true)
.build()));
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).numberOfOCRedPages(numProcessedPages).ocrFinished(true).build());
try {
pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);

View File

@ -3,7 +3,19 @@ package com.iqser.red.service.ocr.v1.server.service;
import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.ocr.v1.api.model.DocumentRequest;
import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
@ -13,37 +25,23 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
@Slf4j
@Service
@RequiredArgsConstructor
public class OcrMessageReceiver {
private final ObjectMapper objectMapper;
private final FileStorageService fileStorageService;
private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
private final OCRService ocrService;
@RabbitHandler
@RabbitListener(queues = MessagingConfiguration.OCR_QUEUE, concurrency = "1")
public void receiveOcr(Message in) throws JsonProcessingException {
public void receiveOcr(Message in) throws IOException {
DocumentRequest ocrRequestMessage = objectMapper.readValue(new String(in.getBody(), StandardCharsets.UTF_8), DocumentRequest.class);
DocumentRequest ocrRequestMessage = objectMapper.readValue(in.getBody(), DocumentRequest.class);
log.info("Start ocr for file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
@ -67,7 +65,7 @@ public class OcrMessageReceiver {
fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
} catch (Exception e) {
log.warn("An exception occurred in ocr file stage: {}", e.getMessage() );
log.warn("An exception occurred in ocr file stage: {}", e.getMessage());
in.getMessageProperties().getHeaders().put(X_ERROR_INFO_HEADER, e.getMessage());
in.getMessageProperties().getHeaders().put(X_ERROR_INFO_TIMESTAMP_HEADER, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
throw e;
@ -77,14 +75,16 @@ public class OcrMessageReceiver {
@RabbitHandler
@RabbitListener(queues = MessagingConfiguration.OCR_DLQ, concurrency = "1")
public void receiveOcrDQL(Message failedMessage) throws JsonProcessingException {
public void receiveOcrDQL(Message failedMessage) throws IOException {
DocumentRequest ocrRequestMessage = objectMapper.readValue(new String(failedMessage.getBody(), StandardCharsets.UTF_8), DocumentRequest.class);
DocumentRequest ocrRequestMessage = objectMapper.readValue(failedMessage.getBody(), DocumentRequest.class);
log.info("OCR DQL received: {}", ocrRequestMessage);
String errorMessage = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp));
fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(),
ocrRequestMessage.getFileId(),
new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp));
}

View File

@ -1,10 +1,7 @@
server:
port: 8086
port: 8097
persistence-service.url: "http://persistence-service-v1:8080"
persistence-service.url: "http://localhost:8085"
tenant-user-management-service.url: "http://localhost:8091/internal"
storage:
bucket-name: 'redaction'
endpoint: 'http://localhost:9000'
key: minioadmin
secret: minioadmin
pdftron.license: demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a

View File

@ -2,6 +2,8 @@ info:
description: OCR Service V1 Server
persistence-service.url: "http://persistence-service-v1:8080"
tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
fforesight.tenants.remote: true
server:
port: 8080
@ -26,9 +28,6 @@ spring:
max-interval: 15000
prefetch: 1
platform.multi-tenancy:
enabled: false
management:
endpoint:

View File

@ -24,6 +24,7 @@ import com.iqser.red.service.ocr.v1.server.initializer.PDFNetInitializer;
import com.iqser.red.service.ocr.v1.server.utils.FileSystemBackedStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.pdftron.pdf.PDFNet;
import jakarta.annotation.PostConstruct;
@ -35,6 +36,9 @@ import lombok.SneakyThrows;
@AutoConfigureObservability
public class AbstractTest {
@MockBean
private TenantsClient tenantsClient;
@Autowired
protected StorageService storageService;

View File

@ -16,10 +16,10 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.ocr.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.ocr.v1.server.service.FileStorageService;
import com.iqser.red.service.ocr.v1.server.service.OCRService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.pdftron.pdf.OCRModule;
import io.micrometer.prometheus.PrometheusMeterRegistry;