Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'feature/RED-10260' into 'master'

Browse Source 
RED-10260: add quoteChar to component mapping

Closes RED-10260

See merge request redactmanager/redaction-service!545
This commit is contained in:
Kilian Schüttler 2024-10-24 11:00:57 +02:00
commit 45cd7f3d98
26 changed files with 128 additions and 595 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.181.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.587.0"
val persistenceServiceVersion = "2.592.0-RED10260.0"
val llmServiceVersion = "1.11.0"
val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4"
@ -52,7 +52,7 @@ dependencies {
implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
implementation("com.iqser.red.commons:storage-commons:2.50.0")
implementation("com.knecon.fforesight:tenant-commons:0.30.0")
implementation("com.knecon.fforesight:tenant-commons:0.31.0")
implementation("com.knecon.fforesight:keycloak-commons:0.30.0") {
exclude(group = "com.knecon.fforesight", module = "tenant-commons")
}

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/PriorityTenantMessagingConfigurationImpl.java
View File

@ -1,12 +0,0 @@
package com.iqser.red.service.redaction.v1.server.configuraton;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
@Configuration
@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
public class PriorityTenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
}

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/TenantMessagingConfigurationImpl.java
View File

@ -1,12 +0,0 @@
package com.iqser.red.service.redaction.v1.server.configuraton;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
@Configuration
@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
}

68
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityTenantExchangeMessageReceiverImpl.java
View File

@ -1,68 +0,0 @@
package com.iqser.red.service.redaction.v1.server.queue;
import static com.iqser.red.service.redaction.v1.model.QueueNames.*;
import java.util.Map;
import java.util.Set;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
@Service
@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
public class PriorityTenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
public PriorityTenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
super(rabbitQueueService, tenantProvider);
}
@Override
protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
return Set.of(TenantQueueConfiguration.builder()
.listenerId(PriorityMessageReceiver.REDACTION_PRIORITY_REQUEST_LISTENER_ID)
.exchangeName(REDACTION_PRIORITY_REQUEST_EXCHANGE)
.queuePrefix(REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX)
.dlqName(REDACTION_DLQ)
.arguments(Map.of("x-max-priority", 2))
.build());
}
@EventListener(ApplicationReadyEvent.class)
public void onApplicationReady() {
System.out.println("application ready invoked");
super.initializeQueues();
}
@RabbitHandler
@RabbitListener(queues = "#{priorityTenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
super.reactToTenantCreation(tenantCreatedEvent);
}
@RabbitHandler
@RabbitListener(queues = "#{priorityTenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
public void reactToTenantDeletion(TenantResponse tenantResponse) {
super.reactToTenantDeletion(tenantResponse);
}
}

68
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/TenantExchangeMessageReceiverImpl.java
View File

@ -1,68 +0,0 @@
package com.iqser.red.service.redaction.v1.server.queue;
import static com.iqser.red.service.redaction.v1.model.QueueNames.*;
import java.util.Map;
import java.util.Set;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
@Service
@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
super(rabbitQueueService, tenantProvider);
}
@Override
protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
return Set.of(TenantQueueConfiguration.builder()
.listenerId(MessageReceiver.REDACTION_REQUEST_LISTENER_ID)
.exchangeName(REDACTION_REQUEST_EXCHANGE)
.queuePrefix(REDACTION_REQUEST_QUEUE_PREFIX)
.dlqName(REDACTION_DLQ)
.arguments(Map.of("x-max-priority", 2))
.build());
}
@EventListener(ApplicationReadyEvent.class)
public void onApplicationReady() {
System.out.println("application ready invoked");
super.initializeQueues();
}
@RabbitHandler
@RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
super.reactToTenantCreation(tenantCreatedEvent);
}
@RabbitHandler
@RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
public void reactToTenantDeletion(TenantResponse tenantResponse) {
super.reactToTenantDeletion(tenantResponse);
}
}

49
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/TenantQueueProviderConfig.java Normal file
View File

@ -0,0 +1,49 @@
package com.iqser.red.service.redaction.v1.server.queue;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DLQ;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_REQUEST_EXCHANGE;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_REQUEST_EXCHANGE;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_REQUEST_QUEUE_PREFIX;
import java.util.Map;
import java.util.Set;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
import com.knecon.fforesight.tenantcommons.model.TenantQueueProvider;
@Configuration
public class TenantQueueProviderConfig {
@Bean
@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
protected TenantQueueProvider getTenantPriorityQueueConfigs() {
return new TenantQueueProvider(Set.of(TenantQueueConfiguration.builder()
.listenerId(PriorityMessageReceiver.REDACTION_PRIORITY_REQUEST_LISTENER_ID)
.exchangeName(REDACTION_PRIORITY_REQUEST_EXCHANGE)
.queuePrefix(REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX)
.dlqName(REDACTION_DLQ)
.arguments(Map.of("x-max-priority", 2))
.build()));
}
@Bean
@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false", matchIfMissing = true)
protected TenantQueueProvider getTenantQueueConfigs() {
return new TenantQueueProvider(Set.of(TenantQueueConfiguration.builder()
.listenerId(MessageReceiver.REDACTION_REQUEST_LISTENER_ID)
.exchangeName(REDACTION_REQUEST_EXCHANGE)
.queuePrefix(REDACTION_REQUEST_QUEUE_PREFIX)
.dlqName(REDACTION_DLQ)
.arguments(Map.of("x-max-priority", 2))
.build()));
}
}

27
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -7,7 +7,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -40,24 +39,16 @@ public class DictionarySearchService {
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
dictionary.getDictionaryModels()
.stream()
.parallel()
.forEach(model -> {
synchronized (node) {
bySearchImplementationAsDictionary(model.getEntriesSearch(),
model.getType(),
model.isHint() ? EntityType.HINT : EntityType.ENTITY,
node,
model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
model.getType(),
EntityType.FALSE_RECOMMENDATION,
node,
model.isDossierDictionary());
if (model.isDossierDictionary()) {
bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
}
bySearchImplementationAsDictionary(model.getEntriesSearch(),
model.getType(),
model.isHint() ? EntityType.HINT : EntityType.ENTITY,
node,
model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
if (model.isDossierDictionary()) {
bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
}
});
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/components/ComponentMappingMemoryCache.java
View File

@ -76,7 +76,7 @@ public class ComponentMappingMemoryCache {
Charset charSet = Charset.forName(metadata.getEncoding());
List<String[]> data;
try (Reader fileReader = new FileReader(mappingFile, charSet); CSVReader reader = buildReader(fileReader, metadata.getDelimiter())) {
try (Reader fileReader = new FileReader(mappingFile, charSet); CSVReader reader = buildReader(fileReader, metadata.getDelimiter(), metadata.getQuoteChar())) {
List<String[]> rawData = reader.readAll();
rawData.remove(0); // remove header row
data = rawData;
@ -86,9 +86,9 @@ public class ComponentMappingMemoryCache {
@SneakyThrows
private static CSVReader buildReader(Reader reader, char delimiter) {
private static CSVReader buildReader(Reader reader, char delimiter, char quoteChar) {
return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build();
return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).withQuoteChar(quoteChar).build()).build();
}

43
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -5,6 +5,7 @@ import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTe
import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_USERNAME;
import static com.knecon.fforesight.tenantcommons.model.TenantResponse.builder;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
@ -30,17 +31,27 @@ import org.junit.jupiter.api.BeforeEach;
import org.mockito.stubbing.Answer;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.rabbit.listener.MessageListenerContainer;
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
import org.springframework.test.context.ContextConfiguration;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@ -64,6 +75,7 @@ import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestConta
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
@ -72,6 +84,7 @@ import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvi
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
@ -85,6 +98,7 @@ import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ContextConfiguration(initializers = {AbstractRedactionIntegrationTest.Initializer.class})
@Import(AbstractRedactionIntegrationTest.TestConfiguration.class)
public abstract class AbstractRedactionIntegrationTest {
protected static final String VERTEBRATE_INDICATOR = "vertebrate";
@ -172,9 +186,6 @@ public abstract class AbstractRedactionIntegrationTest {
@MockBean
protected RabbitAdmin rabbitAdmin;
@MockBean
protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
@MockBean
protected LegalBasisClient legalBasisClient;
@ -739,4 +750,30 @@ public abstract class AbstractRedactionIntegrationTest {
}
}
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class)
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class TestConfiguration {
@Bean
public RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry() {
var mock = mock(RabbitListenerEndpointRegistry.class);
when(mock.getListenerContainer(any())).thenReturn(mock(MessageListenerContainer.class));
return mock;
}
@Bean
@Primary
public StorageService inMemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java
View File

@ -84,7 +84,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
@Import(AbstractRedactionIntegrationTest.TestConfiguration.class)
@Disabled
/*
* This test is meant to be used directly with a download from blob storage (e.g. minio). You need to define the dossier template you want to use by supplying an absolute path.

32
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java
View File

@ -6,42 +6,26 @@ import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
@Import(AnalysisTest.RedactionIntegrationTestConfiguration.class)
public class AnalysisTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/test_rules.drl");
@ -71,22 +55,6 @@ public class AnalysisTest extends AbstractRedactionIntegrationTest {
}
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class)
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void stubClients() {

27
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -17,21 +17,13 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.runtime.KieContainer;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.google.common.collect.Sets;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
@ -40,15 +32,12 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
public class DictionaryServiceTest {
@MockBean
@ -57,9 +46,6 @@ public class DictionaryServiceTest {
@MockBean
protected RabbitAdmin rabbitAdmin;
@MockBean
protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
@MockBean
private TenantsClient tenantsClient;
@ -81,19 +67,6 @@ public class DictionaryServiceTest {
@Autowired
protected DictionaryService dictionaryService;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
public static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@Test
public void testDictionaryServiceConsistency() {

48
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -19,19 +18,10 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@ -40,21 +30,15 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
@Import(DocumineFloraTest.RedactionIntegrationTestConfiguration.class)
public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/documine_flora.drl");
@ -192,22 +176,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
}
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class)
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void stubClients() {
@ -277,9 +245,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
.fileId(TEST_FILE_ID)
.build());
});
request.setManualRedactions(ManualRedactions.builder()
.idsToRemove(idRemovalSet)
.build());
request.setManualRedactions(ManualRedactions.builder().idsToRemove(idRemovalSet).build());
analyzeService.reanalyze(request);
@ -307,12 +273,20 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished analysis");
var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var doseMortality = componentLog.getComponentLogEntries().stream().filter(componentLogEntry -> componentLogEntry.getName().equals("Dose_Mortality")).findFirst().get();
var doseMortality = componentLog.getComponentLogEntries()
.stream()
.filter(componentLogEntry -> componentLogEntry.getName().equals("Dose_Mortality"))
.findFirst()
.get();
assertEquals(doseMortality.getValues().size(), 5);
Pattern pattern = Pattern.compile("^5000, [SD]$");
boolean allMatch = doseMortality.getValues().stream().map(ComponentLogEntryValue::getValue).allMatch(pattern.asPredicate());
boolean allMatch = doseMortality.getValues()
.stream()
.map(ComponentLogEntryValue::getValue)
.allMatch(pattern.asPredicate());
assertTrue(allMatch);
}
}

28
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MapEntityLogToDocumentDataTest.java
View File

@ -24,15 +24,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -53,15 +45,11 @@ import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ -71,7 +59,6 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
public class MapEntityLogToDocumentDataTest extends AbstractRedactionIntegrationTest {
public static final int MATCH_THRESHOLD = 25;
@ -91,21 +78,6 @@ public class MapEntityLogToDocumentDataTest extends AbstractRedactionIntegration
private AtomicLong totalNotFoundEntities;
private AtomicLong totalRemovedDuplicates;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({LayoutParsingServiceProcessorConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void stubClients() {

31
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -18,19 +17,10 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@ -45,24 +35,18 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
@ -70,21 +54,6 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
@Autowired
DictionaryService dictionaryService;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class)
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void stubClients() {

38
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -6,8 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.BufferedWriter;
@ -37,21 +35,11 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.type.TypeReference;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@ -83,12 +71,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.storage.DocumentDataFallbackService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.utils.ProtobufUtil;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -96,29 +79,12 @@ import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class RedactionIntegrationTest extends RulesIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules.drl");
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({LayoutParsingServiceProcessorConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void stubClients() {
@ -2252,9 +2218,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
@Test
public void testFileWithImagesAndNoText() {
AnalyzeRequest request = prepareStorage("files/new/only_images.pdf",
"files/cv_service_empty_response.json",
"files/only_images_file_image_response.json");
AnalyzeRequest request = prepareStorage("files/new/only_images.pdf", "files/cv_service_empty_response.json", "files/only_images_file_image_response.json");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
AnalyzeResult result = analyzeService.analyze(request);

30
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -10,30 +10,16 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ -41,26 +27,10 @@ import lombok.SneakyThrows;
@Disabled // TODO: re-enable, when redaction log is removed
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class)
public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules_v2.drl");
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class)
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void stubClients() {

32
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java
View File

@ -1,52 +1,22 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(BuildDocumentIntegrationTest.TestConfiguration.class)
public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTest {
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class)
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class TestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@SneakyThrows
protected Document buildGraph(String filename) {

32
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -16,17 +16,9 @@ import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.kie.internal.io.ResourceFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@ -45,6 +37,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@ -54,8 +47,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@Import(DocumentPerformanceIntegrationTest.TestConfiguration.class)
public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegrationTest {
public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest {
private static final String RULES = "drools/rules.drl";
@ -73,26 +65,6 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
@Autowired
private KieContainer kieContainer;
@Configuration
@Import(BuildDocumentIntegrationTest.TestConfiguration.class)
public static class TestConfiguration {
@Bean
public KieContainer kieContainer() {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.write(ResourceFactory.newClassPathResource(RULES));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();
return kieServices.newKieContainer(kieModule.getReleaseId());
}
}
@BeforeEach
public void stubClients() {

30
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -24,18 +24,9 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@ -53,7 +44,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.Application;
@ -65,11 +55,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ -77,7 +63,6 @@ import lombok.SneakyThrows;
@Disabled // TODO: only disabled due to OOM in gradle testRunner in GitLab, find a solution!
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(ManualChangesEnd2EndTest.TestConfiguration.class)
public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
@ -87,21 +72,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
private EntityCreationService entityCreationService;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class)
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class TestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void createServices() {

32
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -20,19 +20,11 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.ResourcePatternResolver;
@ -41,12 +33,12 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@ -58,9 +50,6 @@ import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
@ -70,7 +59,7 @@ import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
@Import(AbstractRedactionIntegrationTest.TestConfiguration.class)
@AutoConfigureObservability
public class LiveDataIntegrationTest {
@ -88,9 +77,6 @@ public class LiveDataIntegrationTest {
@MockBean
protected RabbitAdmin rabbitAdmin;
@MockBean
protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
@MockBean
private TenantsClient tenantsClient;
@ -135,20 +121,6 @@ public class LiveDataIntegrationTest {
private List<Type> types;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@SneakyThrows
@BeforeEach

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java
View File

@ -19,6 +19,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@ -27,6 +28,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityCreation
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@ExtendWith(MockitoExtension.class)
@Import(RulesIntegrationTest.TestConfiguration.class)
public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
protected static final String RULES = "drools/rules.drl";
@ -55,7 +57,7 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
@Configuration
@Import(BuildDocumentIntegrationTest.TestConfiguration.class)
@Import(AbstractRedactionIntegrationTest.TestConfiguration.class)
public static class TestConfiguration {
@Bean

30
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentSearchServiceTest.java
View File

@ -14,19 +14,10 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse;
@ -36,38 +27,17 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
public class DocumentSearchServiceTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({LayoutParsingServiceProcessorConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@Captor
ArgumentCaptor<BulkLocalResponse> captor;

32
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.service.document;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@ -21,19 +20,10 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@ -41,44 +31,22 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({LayoutParsingServiceProcessorConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@Captor
ArgumentCaptor<AnalyzeResponse> captor;

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/mappings/GuidelineMapping.csv.metadata.json
View File

@ -11,5 +11,6 @@
"storageId": "2c4cfafc-b3c3-48de-8706-a979dea764aa/f48e8ab0-250b-4a73-aa4d-436a7176841f_GuidelineMapping_GuidelineMapping.csv",
"numberOfLines": 30,
"encoding": "UTF-8",
"delimiter": ","
"delimiter": ",",
"quoteChar": "\""
}

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/mappings/GuidelineMapping2.csv.metadata.json
View File

@ -11,5 +11,6 @@
"storageId": "2c4cfafc-b3c3-48de-8706-a979dea764aa/f48e8ab0-250b-4a73-aa4d-436a7176841f_GuidelineMapping_GuidelineMapping.csv",
"numberOfLines": 28,
"encoding": "UTF-8",
"delimiter": ","
"delimiter": ",",
"quoteChar": "\""
}