Project structure cleanup / adapted to fforesight modules

This commit is contained in:
Timo Bejan 2023-07-12 20:46:01 +03:00
parent df9cbdc036
commit 4fc7bac818
19 changed files with 44 additions and 476 deletions

View File

@ -17,7 +17,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
<version>${guava.version}</version>
</dependency>
</dependencies>

View File

@ -14,9 +14,14 @@
<dependencies>
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>persistence-service-internal-api-v1</artifactId>
<artifactId>persistence-service-shared-api-v1</artifactId>
<version>2.36.0</version>
</dependency>
<dependency>
<groupId>com.knecon.fforesight</groupId>
<artifactId>tenant-commons</artifactId>
<version>${tennat-commons.version}</version>
</dependency>
<dependency>
<groupId>com.knecon.fforesight</groupId>
<artifactId>layoutparser-service-internal-api</artifactId>
@ -25,12 +30,7 @@
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>storage-commons</artifactId>
<version>1.13.0</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>spring-commons</artifactId>
<version>6.2.0</version>
<version>${storage-commons.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
@ -42,11 +42,6 @@
<artifactId>pdfbox-tools</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-afterburner</artifactId>
@ -57,67 +52,14 @@
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
<version>4.0.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-commons</artifactId>
</dependency>
</dependencies>
<repositories>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<releases>
<enabled>false</enabled>
</releases>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<releases>
<enabled>false</enabled>
</releases>
</pluginRepository>
</pluginRepositories>
</project>

View File

@ -22,7 +22,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.PageData;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

View File

@ -1,27 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import java.util.concurrent.Executor;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
public class AsyncConfig extends AsyncConfigurerSupport {
@Override
public Executor getAsyncExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(7);
executor.setMaxPoolSize(42);
executor.setQueueCapacity(11);
executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
executor.setTaskDecorator(new TenantAwareTaskDecorator());
executor.initialize();
return executor;
}
}

View File

@ -1,105 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import java.util.Base64;
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.SecretKeySpec;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import lombok.SneakyThrows;
@Service
public class EncryptionDecryptionService {
@Value("${redaction-service.crypto.key:redaction}")
private String key;
private SecretKey secretKey;
private byte[] iv;
@SneakyThrows
@PostConstruct
protected void postConstruct() {
SecureRandom secureRandom = new SecureRandom();
iv = new byte[12];
secureRandom.nextBytes(iv);
secretKey = generateSecretKey(key, iv);
}
@SneakyThrows
public String encrypt(String strToEncrypt) {
return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
}
@SneakyThrows
public String decrypt(String strToDecrypt) {
byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
return new String(decrypt(bytes), StandardCharsets.UTF_8);
}
@SneakyThrows
public byte[] encrypt(byte[] data) {
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
byte[] encryptedData = cipher.doFinal(data);
ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
byteBuffer.putInt(iv.length);
byteBuffer.put(iv);
byteBuffer.put(encryptedData);
return byteBuffer.array();
}
@SneakyThrows
public byte[] decrypt(byte[] encryptedData) {
ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
int noonceSize = byteBuffer.getInt();
if (noonceSize < 12 || noonceSize >= 16) {
throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
}
byte[] iv = new byte[noonceSize];
byteBuffer.get(iv);
SecretKey secretKey = generateSecretKey(key, iv);
byte[] cipherBytes = new byte[byteBuffer.remaining()];
byteBuffer.get(cipherBytes);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
return cipher.doFinal(cipherBytes);
}
@SneakyThrows
public SecretKey generateSecretKey(String password, byte[] iv) {
KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
return new SecretKeySpec(key, "AES");
}
}

View File

@ -1,17 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import org.springframework.stereotype.Component;
import feign.RequestInterceptor;
import feign.RequestTemplate;
@Component
public class ForwardTenantInterceptor implements RequestInterceptor {
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
@Override
public void apply(RequestTemplate template) {
template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
}
}

View File

@ -1,48 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class MultiTenancyMessagingConfiguration {
@Bean
public static BeanPostProcessor multitenancyBeanPostProcessor() {
return new BeanPostProcessor() {
@Override
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
if (bean instanceof RabbitTemplate) {
((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
m.getMessageProperties().setHeader(ForwardTenantInterceptor.TENANT_HEADER_NAME, TenantContext.getTenantId());
return m;
});
} else if (bean instanceof AbstractRabbitListenerContainerFactory) {
((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
String tenant = m.getMessageProperties().getHeader(ForwardTenantInterceptor.TENANT_HEADER_NAME);
if (tenant != null) {
TenantContext.setTenantId(tenant);
} else {
throw new RuntimeException("No Tenant is set queue message");
}
return m;
});
}
return bean;
}
};
}
}

View File

@ -1,28 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
@Configuration
public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
private final TenantInterceptor tenantInterceptor;
@Autowired
public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
this.tenantInterceptor = tenantInterceptor;
}
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addWebRequestInterceptor(tenantInterceptor);
}
}

View File

@ -1,44 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import org.springframework.stereotype.Service;
import com.iqser.red.storage.commons.model.AzureStorageConnection;
import com.iqser.red.storage.commons.model.S3StorageConnection;
import com.iqser.red.storage.commons.service.StorageConnectionProvider;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class StorageConnectionProviderImpl implements StorageConnectionProvider {
private final TenantsClient tenantsClient;
private final EncryptionDecryptionService encryptionDecryptionService;
@Override
public AzureStorageConnection getAzureStorageConnection(String tenantId) {
var tenant = tenantsClient.getTenant(tenantId);
return AzureStorageConnection.builder()
.connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
.containerName(tenant.getAzureStorageConnection().getContainerName())
.build();
}
@Override
public S3StorageConnection getS3StorageConnection(String tenantId) {
var tenant = tenantsClient.getTenant(tenantId);
return S3StorageConnection.builder()
.key(tenant.getS3StorageConnection().getKey())
.secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
.signerType(tenant.getS3StorageConnection().getSignerType())
.bucketName(tenant.getS3StorageConnection().getBucketName())
.region(tenant.getS3StorageConnection().getRegion())
.endpoint(tenant.getS3StorageConnection().getEndpoint())
.build();
}
}

View File

@ -1,23 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import org.springframework.core.task.TaskDecorator;
import org.springframework.lang.NonNull;
public class TenantAwareTaskDecorator implements TaskDecorator {
@Override
@NonNull
public Runnable decorate(@NonNull Runnable runnable) {
String tenantId = TenantContext.getTenantId();
return () -> {
try {
TenantContext.setTenantId(tenantId);
runnable.run();
} finally {
TenantContext.setTenantId(null);
}
};
}
}

View File

@ -1,29 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public final class TenantContext {
private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
public static void setTenantId(String tenantId) {
log.debug("Setting tenantId to " + tenantId);
currentTenant.set(tenantId);
}
public static String getTenantId() {
return currentTenant.get();
}
public static void clear() {
currentTenant.remove();
}
}

View File

@ -1,35 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import org.springframework.stereotype.Component;
import org.springframework.ui.ModelMap;
import org.springframework.web.context.request.WebRequest;
import org.springframework.web.context.request.WebRequestInterceptor;
@Component
public class TenantInterceptor implements WebRequestInterceptor {
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
@Override
public void preHandle(WebRequest request) {
if (request.getHeader(TENANT_HEADER_NAME) != null) {
TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
}
}
@Override
public void postHandle(WebRequest request, ModelMap model) {
TenantContext.clear();
}
@Override
public void afterCompletion(WebRequest request, Exception ex) {
}
}

View File

@ -1,10 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.multitenancy;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
public interface TenantsClient extends TenantsResource {
}

View File

@ -18,24 +18,16 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>spring-commons</artifactId>
<version>6.2.0</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>4.0.1</version>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
<version>4.0.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
<version>${spring.version}</version>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>

View File

@ -2,22 +2,19 @@ package com.knecon.fforesight.service.layoutparser.server;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Import;
import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.AsyncConfig;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.MultiTenancyMessagingConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.MultiTenancyWebConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.TenantsClient;
import com.knecon.fforesight.service.layoutparser.processor.queue.MessagingConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class, MetricsConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, StorageAutoConfiguration.class, MessagingConfiguration.class})
@EnableFeignClients(basePackageClasses = TenantsClient.class)
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
public class Application {

View File

@ -1,7 +1,7 @@
server:
port: 8083
persistence-service.url: "http://localhost:8085"
tenant-user-management-service.url: "http://localhost:8091/internal"
storage:
bucket-name: 'redaction'
@ -9,9 +9,3 @@ storage:
key: minioadmin
secret: minioadmin
redaction-service:
enableImageClassification: false
cvTableParsingEnabled: false
nerServiceEnabled: false
priorityMode: false

View File

@ -1,7 +1,8 @@
info:
description: Layout Parser Service Processor
persistence-service.url: "http://persistence-service-v1:8080"
tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
fforesight.tenants.remote: true
server:
port: 8080
@ -9,8 +10,6 @@ server:
spring:
main:
allow-circular-references: true # FIXME
profiles:
active: kubernetes
rabbitmq:
host: ${RABBITMQ_HOST:localhost}
port: ${RABBITMQ_PORT:5672}

View File

@ -22,10 +22,10 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.TenantContext;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.TenantsClient;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.server.Application;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import lombok.SneakyThrows;
@ -40,7 +40,7 @@ public class BaseTest {
@Autowired
protected StorageService storageService;
@Autowired
@MockBean
protected TenantsClient tenantsClient;
@MockBean

View File

@ -4,10 +4,9 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.iqser.red</groupId>
<artifactId>platform-dependency</artifactId>
<version>2.3.0</version>
<relativePath/>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.0.6</version>
</parent>
<groupId>com.knecon.fforesight</groupId>
@ -24,26 +23,37 @@
<properties>
<java.version>17</java.version>
<jackson.version>2.13.2</jackson.version>
<slf4j.version>2.0.7</slf4j.version>
<pdfbox.version>3.0.0-alpha2</pdfbox.version>
<spring.version>3.0.1</spring.version>
<spring.cloud.version>2022.0.1</spring.cloud.version>
<guava.version>31.1-jre</guava.version>
<jackson.version>2.15.0-rc2</jackson.version>
<tennat-commons.version>0.10.0</tennat-commons.version>
<storage-commons.version>2.1.0</storage-commons.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.28</version>
<scope>provided</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-ui</artifactId>
<version>1.6.13</version>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>2022.0.2</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<pluginManagement>
<plugins>