This commit is contained in:
maverickstuder 2024-03-12 17:56:50 +01:00
parent e79bcbbc10
commit 4a22f97a88
13 changed files with 474 additions and 3 deletions

View File

@ -56,12 +56,14 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}")
implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
implementation("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}")
implementation("net.logstash.logback:logstash-logback-encoder:7.4")
implementation("ch.qos.logback:logback-classic")
implementation("org.reflections:reflections:0.10.2")
testImplementation(project(":rules-management"))
testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")

View File

@ -1,5 +1,11 @@
package com.iqser.red.service.redaction.v1.server;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.Date;
import org.bson.Document;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
@ -10,6 +16,9 @@ import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.core.convert.converter.Converter;
import org.springframework.data.mongodb.core.convert.MongoCustomConversions;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@ -27,6 +36,7 @@ import io.micrometer.observation.aop.ObservedAspect;
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
@EnableMongoRepositories
public class Application {
public static void main(String[] args) {
@ -63,4 +73,41 @@ public class Application {
return new DeprecatedElementsFinder();
}
@Bean
public MongoCustomConversions mongoCustomConversions() {
return new MongoCustomConversions(Arrays.asList(new MongoOffsetDateTimeWriter(), new MongoOffsetDateTimeReader()));
}
public static class MongoOffsetDateTimeWriter implements Converter<OffsetDateTime, Document> {
public static final String DATE_FIELD = "dateTime";
public static final String OFFSET_FIELD = "offset";
@Override
public Document convert(final OffsetDateTime offsetDateTime) {
final Document document = new Document();
document.put(DATE_FIELD, Date.from(offsetDateTime.toInstant()));
document.put(OFFSET_FIELD, offsetDateTime.getOffset().toString());
return document;
}
}
public static class MongoOffsetDateTimeReader implements Converter<Document, OffsetDateTime> {
@Override
public OffsetDateTime convert(final Document document) {
final Date dateTime = document.getDate(MongoOffsetDateTimeWriter.DATE_FIELD);
final ZoneOffset offset = ZoneOffset.of(document.getString(MongoOffsetDateTimeWriter.OFFSET_FIELD));
return OffsetDateTime.ofInstant(dateTime.toInstant(), offset);
}
}
}

View File

@ -0,0 +1,54 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Document(collection = "entity-logs")
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class EntityLogDocument {
@Id
private String id;
private EntityLog entityLog;
/**
private long analysisVersion;
private int analysisNumber;
private List<EntityLogEntry> entityLogEntry = new ArrayList<>();
private List<EntityLogLegalBasis> legalBasis = new ArrayList<>();
private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1;
public EntityLogDocument(String id, EntityLog entityLog) {
this.id = id;
this.analysisVersion = entityLog.getAnalysisVersion();
this.analysisNumber = entityLog.getAnalysisNumber();
this.entityLogEntry = entityLog.getEntityLogEntry();
this.legalBasis = entityLog.getLegalBasis();
this.dictionaryVersion = entityLog.getDictionaryVersion();
this.dossierDictionaryVersion = entityLog.getDossierDictionaryVersion();
this.rulesVersion = entityLog.getRulesVersion();
this.legalBasisVersion = entityLog.getLegalBasisVersion();
} **/
}

View File

@ -0,0 +1,7 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import org.springframework.data.mongodb.repository.MongoRepository;
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {
}

View File

@ -0,0 +1,34 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.stereotype.Service;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
public class EntityLogDocumentService {
private final EntityLogDocumentRepository entityLogDocumentRepository;
public EntityLogDocumentService(EntityLogDocumentRepository entityLogDocumentRepository) {this.entityLogDocumentRepository = entityLogDocumentRepository;}
public EntityLogDocument saveEntityLogDocument(EntityLogDocument entityLogDocument) {
return entityLogDocumentRepository.save(entityLogDocument);
}
public Optional<EntityLogDocument> findEntityLogDocumentById(String id) {
return entityLogDocumentRepository.findById(id);
}
public boolean entityLogDocumentExists(String id) {
return entityLogDocumentRepository.existsById(id);
}
}

View File

@ -0,0 +1,24 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Repository;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Repository
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@RequiredArgsConstructor
public class _EntityLogDocumentRepository {
MongoTemplate mongoTemplate;
public EntityLogDocument saveEntityLogDocument(EntityLogDocument entityLogDocument) {
return mongoTemplate.save(entityLogDocument);
}
public EntityLogDocument findEntityLogDocumentById(String id) {
return mongoTemplate.findById(id, EntityLogDocument.class);
}
}

View File

@ -39,6 +39,8 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncr
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
@ -79,6 +81,7 @@ public class AnalyzeService {
ImportedRedactionEntryService importedRedactionEntryService;
ObservedStorageService observedStorageService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
EntityLogDocumentService entityLogDocumentService;
@Timed("redactmanager_reanalyze")
@ -255,7 +258,8 @@ public class AnalyzeService {
Set<FileAttribute> addedFileAttributes) {
EntityLog entityLog = entityLogChanges.getEntityLog();
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
entityLogDocumentService.saveEntityLogDocument(new EntityLogDocument(analyzeRequest.getFileId(), entityLog));
//redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (entityLogChanges.isHasChanges() || !isReanalysis) {

View File

@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentService;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
@ -39,6 +40,8 @@ public class RedactionStorageService {
private final StorageService storageService;
private final EntityLogDocumentService entityLogDocumentService;
@SneakyThrows
public InputStream getStoredObject(String storageId) {
@ -132,7 +135,8 @@ public class RedactionStorageService {
public EntityLog getEntityLog(String dossierId, String fileId) {
try {
EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
//EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
EntityLog entityLog = entityLogDocumentService.findEntityLogDocumentById(fileId).orElseThrow(() -> new StorageObjectDoesNotExist("")).getEntityLog();
entityLog.setEntityLogEntry(entityLog.getEntityLogEntry()
.stream()
.filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty()))
@ -200,7 +204,8 @@ public class RedactionStorageService {
public boolean entityLogExists(String dossierId, String fileId) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG));
//return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG));
return entityLogDocumentService.entityLogDocumentExists(fileId);
}

View File

@ -46,6 +46,13 @@ spring:
port: ${REDIS_PORT:6379}
password: ${REDIS_PASSWORD}
timeout: 60000
mongodb:
# todo: multi-tenancy
database: redaction
host: ${MONGODB_HOST:localhost}
port: 27017
username: ${MONGODB_USER}
password: ${MONGODB_PASSWORD}
management:
endpoint:

View File

@ -0,0 +1,86 @@
package com.iqser.red.service.redaction.v1.server;
import static com.mongodb.client.model.Filters.eq;
import org.bson.Document;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoProperties;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.data.mongodb.config.AbstractMongoClientConfiguration;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentRepository;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
public class MyMongoTest {
private static final String HOSTNAME = "localhost";
private static final String PORT = "27017";
private static final String DATABASE = "redaction";
private static final String USERNAME = "root";
private static final String PASSWORD = "a2sm7FUBAfLAjfIm1KubeFMAsrIt1pF4";
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
public static final String COLLECTION_ENTITY_LOG = "entity-log";
private static final String CONNECTION_STRING = String.format("mongodb://%s:%s@%s:%s/", USERNAME, PASSWORD, HOSTNAME, PORT);
@Test
public void testMongoDB() {
try (MongoClient mongoClient = MongoClients.create(CONNECTION_STRING)) {
MongoDatabase database = mongoClient.getDatabase(DATABASE);
MongoCollection<Document> collection = database.getCollection(COLLECTION_ENTITY_LOG);
Document doc = collection.find(eq("id", TEST_FILE_ID)).first();
if (doc != null) {
System.out.println("Found doc with _id: " + doc.get("_id"));
} else {
System.out.println("No matching documents found.");
}
}
}
@Test
public void testMongoDB_Template() {
try (MongoClient mongoClient = MongoClients.create(CONNECTION_STRING)) {
MongoTemplate mongoTemplate = new MongoTemplate(mongoClient, DATABASE);
MongoCollection<Document> collection = mongoTemplate.getCollection(COLLECTION_ENTITY_LOG);
Document doc = collection.find(eq("id", TEST_FILE_ID)).first();
if (doc != null) {
System.out.println("Found doc with _id: " + doc.get("_id"));
} else {
System.out.println("No matching documents found.");
}
}
}
}

View File

@ -0,0 +1,192 @@
package com.iqser.red.service.redaction.v1.server;
import static com.mongodb.client.model.Filters.eq;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.InstanceOfAssertFactories.FILE;
import static org.mockito.ArgumentMatchers.any;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.Date;
import java.util.Optional;
import org.bson.Document;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.runtime.KieContainer;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.convert.converter.Converter;
import org.springframework.core.io.ClassPathResource;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.convert.MongoCustomConversions;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentRepository;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(MyOtherMongoTest.MongoTestConfiguration.class)
@ContextConfiguration(initializers = {MyOtherMongoTest.Initializer.class})
public class MyOtherMongoTest {
@MockBean
private RabbitTemplate rabbitTemplate;
@MockBean
private TenantsClient tenantsClient;
@MockBean
protected KieContainer kieContainer;
@MockBean
protected DictionaryClient dictionaryClient;
private static final String HOSTNAME = "localhost";
private static final String PORT = "27017";
private static final String DATABASE = "redaction";
private static final String USERNAME = "user";
private static final String PASSWORD = "password";
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
public static final String COLLECTION_ENTITY_LOG = "entity-log";
private static final String CONNECTION_STRING = String.format("mongodb://%s:%s@%s:%s/", USERNAME, PASSWORD, HOSTNAME, PORT);
@Autowired
private EntityLogDocumentRepository entityLogDocumentRepository;
@Autowired
private EntityLogDocumentService entityLogDocumentService;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
public static class RedactionIntegrationTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@Test
@SneakyThrows
public void testEntityLogDocumentRepo() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_FILE_ID, entityLog);
entityLogDocumentService.saveEntityLogDocument(entityLogDocument);
Optional<EntityLogDocument> found = entityLogDocumentService.findEntityLogDocumentById(TEST_FILE_ID);
assert(found.isPresent());
}
@Configuration
@EnableMongoRepositories
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({LayoutParsingServiceProcessorConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class MongoTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
@Bean
public MongoCustomConversions mongoCustomConversions() {
return new MongoCustomConversions(Arrays.asList(new MongoOffsetDateTimeWriter(), new MongoOffsetDateTimeReader()
));
}
}
public static class MongoOffsetDateTimeWriter implements Converter<OffsetDateTime, Document> {
public static final String DATE_FIELD = "dateTime";
public static final String OFFSET_FIELD = "offset";
@Override
public Document convert(final OffsetDateTime offsetDateTime) {
final Document document = new Document();
document.put(DATE_FIELD, Date.from(offsetDateTime.toInstant()));
document.put(OFFSET_FIELD, offsetDateTime.getOffset().toString());
return document;
}
}
public static class MongoOffsetDateTimeReader implements Converter<Document, OffsetDateTime> {
@Override
public OffsetDateTime convert(final Document document) {
final Date dateTime = document.getDate(MongoOffsetDateTimeWriter.DATE_FIELD);
final ZoneOffset offset = ZoneOffset.of(document.getString(MongoOffsetDateTimeWriter.OFFSET_FIELD));
return OffsetDateTime.ofInstant(dateTime.toInstant(), offset);
}
}
@Slf4j
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
TestPropertyValues.of("MONGODB_HOST=" + HOSTNAME, "MONGODB_PORT=" + PORT, "MONGODB_USERNAME=" + USERNAME, "MONGODB_PASSWORD=" + PASSWORD)
.applyTo(configurableApplicationContext.getEnvironment());
}
}
}

View File

@ -12,6 +12,14 @@ spring:
allow-circular-references: true # FIXME
cache:
type: NONE
data:
mongodb:
# todo: multi-tenancy
database: redaction
host: ${MONGODB_HOST:localhost}
port: ${MONGODB_PORT:27017}
username: ${MONGODB_USER}
password: ${MONGODB_PASSWORD}
processing.kafkastreams: false