Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* pre-test eval version
This commit is contained in:
maverickstuder 2024-03-19 15:33:52 +01:00
parent 14a219df75
commit c9f2739cb1
10 changed files with 17 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java
View File

@ -24,7 +24,7 @@ import io.micrometer.observation.aop.ObservedAspect;
@EnableCaching
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDataSources.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})

79
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogDocument.java
View File

@ -1,79 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.util.ArrayList;
import java.util.List;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.IndexDirection;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@Document(collection = "entity-logs")
public class EntityLogDocument {
@Id
@EqualsAndHashCode.Include
private String id;
private String dossierId;
private String fileId;
private long analysisVersion;
//@Indexed(direction = IndexDirection.DESCENDING)
private int analysisNumber;
@DBRef
private List<EntityLogEntryDocument> entityLogEntryDocument = new ArrayList<>();
private List<EntityLogLegalBasis> legalBasis = new ArrayList<>();
private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1;
public EntityLogDocument(String dossierId, String fileId, EntityLog entityLog) {
this.id = getDocumentId(dossierId, fileId);
this.dossierId = dossierId;
this.fileId = fileId;
this.analysisVersion = entityLog.getAnalysisVersion();
this.analysisNumber = entityLog.getAnalysisNumber();
this.entityLogEntryDocument = new ArrayList<>(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(this.id, entityLogEntry)).toList());
this.legalBasis = entityLog.getLegalBasis();
this.dictionaryVersion = entityLog.getDictionaryVersion();
this.dossierDictionaryVersion = entityLog.getDossierDictionaryVersion();
this.rulesVersion = entityLog.getRulesVersion();
this.legalBasisVersion = entityLog.getLegalBasisVersion();
}
@NotNull
public static String getDocumentId(String dossierId, String fileId) {
return dossierId + "/" + fileId;
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogDocumentRepository.java
View File

@ -6,6 +6,8 @@ import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.data.repository.query.Param;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {
@Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }")

116
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogEntryDocument.java
View File

@ -1,116 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.CompoundIndex;
import org.springframework.data.mongodb.core.index.CompoundIndexes;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.data.redis.core.index.Indexed;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
@Document(collection = "entity-log-entries")
/**@CompoundIndexes({
@CompoundIndex(name = "changes_analysisNumber", def = "{ 'changes.analysisNumber': 1 }")
})**/ public class EntityLogEntryDocument {
@Id
@EqualsAndHashCode.Include
String id;
String entryId;
String entityLogId;
String type;
EntryType entryType;
EntryState state;
String value;
String reason;
String matchedRule;
String legalBasis;
List<Integer> containingNodeId;
String closestHeadline;
String section;
List<Position> positions = new ArrayList<>();
String textBefore;
String textAfter;
int startOffset;
int endOffset;
boolean imageHasTransparency;
boolean dictionaryEntry;
boolean dossierDictionaryEntry;
boolean excluded;
List<Change> changes = new ArrayList<>();
List<ManualChange> manualChanges = new ArrayList<>();
Set<Engine> engines = new HashSet<>();
Set<String> reference = new HashSet<>();
Set<String> importedRedactionIntersections = new HashSet<>();
int numberOfComments;
public EntityLogEntryDocument(String entityLogId, EntityLogEntry entityLogEntry) {
this.id = entityLogId + "/" + entityLogEntry.getId();
this.entryId = entityLogEntry.getId();
this.entityLogId = entityLogId;
this.type = entityLogEntry.getType();
this.entryType = entityLogEntry.getEntryType();
this.state = entityLogEntry.getState();
this.value = entityLogEntry.getValue();
this.reason = entityLogEntry.getReason();
this.matchedRule = entityLogEntry.getMatchedRule();
this.legalBasis = entityLogEntry.getLegalBasis();
this.containingNodeId = new ArrayList<>(entityLogEntry.getContainingNodeId());
this.closestHeadline = entityLogEntry.getClosestHeadline();
this.section = entityLogEntry.getSection();
this.positions = new ArrayList<>(entityLogEntry.getPositions());
this.textBefore = entityLogEntry.getTextBefore();
this.textAfter = entityLogEntry.getTextAfter();
this.startOffset = entityLogEntry.getStartOffset();
this.endOffset = entityLogEntry.getEndOffset();
this.imageHasTransparency = entityLogEntry.isImageHasTransparency();
this.dictionaryEntry = entityLogEntry.isDictionaryEntry();
this.dossierDictionaryEntry = entityLogEntry.isDossierDictionaryEntry();
this.excluded = entityLogEntry.isExcluded();
this.changes = new ArrayList<>(entityLogEntry.getChanges());
this.manualChanges = new ArrayList<>(entityLogEntry.getManualChanges());
this.engines = new HashSet<>(entityLogEntry.getEngines());
this.reference = new HashSet<>(entityLogEntry.getReference());
this.importedRedactionIntersections = new HashSet<>(entityLogEntry.getImportedRedactionIntersections());
this.numberOfComments = entityLogEntry.getNumberOfComments();
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogEntryDocumentRepository.java
View File

@ -5,6 +5,8 @@ import java.util.List;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
public interface EntityLogEntryDocumentRepository extends MongoRepository<EntityLogEntryDocument, String> {
@Query("{ 'entityLogId' : ?0, 'manualChanges' : { $exists: true, $not: { $size: 0 } } }")

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java
View File

@ -9,6 +9,8 @@ import java.util.stream.Collectors;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoLiquibaseTest.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server;
import static com.mongodb.client.model.Filters.eq;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import liquibase.Contexts;
@ -22,6 +23,7 @@ public class MyMongoLiquibaseTest {
private static final String URL = String.format("mongodb://%s:%s/%s", HOSTNAME, PORT, DATABASE);
@Test
@Disabled
@SneakyThrows
public void testMongoDB() {

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoTest.java
View File

@ -6,6 +6,7 @@ import org.bson.BsonArray;
import org.bson.BsonDocument;
import org.bson.BsonString;
import org.bson.Document;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.data.mongodb.core.MongoTemplate;
@ -15,6 +16,7 @@ import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
@Disabled
public class MyMongoTest {
private static final String HOSTNAME = "localhost";

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java
View File

@ -12,6 +12,7 @@ import java.util.List;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.runtime.KieContainer;
@ -31,18 +32,16 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.core.io.ClassPathResource;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.service.StorageServiceImpl;
import com.knecon.fforesight.mongo.database.commons.config.MongoDbConfiguration;
import com.knecon.fforesight.mongo.database.commons.config.MultiTenantMongoDBFactory;
@ -63,6 +62,7 @@ import lombok.extern.slf4j.Slf4j;
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(MyOtherMongoTest.MongoTestConfiguration.class)
@ContextConfiguration(initializers = {MyOtherMongoTest.Initializer.class})
@Disabled
public class MyOtherMongoTest {
@MockBean

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml
View File

@ -44,3 +44,6 @@ management:
health.enabled: true
endpoints.web.exposure.include: prometheus, health, metrics
metrics.export.prometheus.enabled: true
logging.level.root: debug