Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* added new CRUD operations to repositories and services
* removed cascade logic as updates and deletes are not trivial to implement, and thus it is better to do all CRUD operations alike
This commit is contained in:
maverickstuder 2024-03-18 13:48:35 +01:00
parent 8f064c34fb
commit 6ba00325d3
18 changed files with 120724 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java
View File

@ -1,11 +1,5 @@
package com.iqser.red.service.redaction.v1.server;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.Date;
import org.bson.Document;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
@ -16,15 +10,10 @@ import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.core.convert.converter.Converter;
import org.springframework.data.mongodb.core.convert.MongoCustomConversions;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.mongo.CascadeSaveMongoEventListener;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.service.MongoClientCache;
import com.knecon.fforesight.mongo.database.commons.service.MongoDataSources;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@ -39,7 +28,6 @@ import io.micrometer.observation.aop.ObservedAspect;
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
@EnableMongoRepositories
public class Application {
public static void main(String[] args) {
@ -76,11 +64,4 @@ public class Application {
return new DeprecatedElementsFinder();
}
@Bean
public CascadeSaveMongoEventListener cascadingMongoEventListener() {
return new CascadeSaveMongoEventListener();
}
}

54
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/CascadeCallback.java
View File

@ -1,54 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.lang.reflect.Field;
import java.lang.reflect.InaccessibleObjectException;
import java.util.Collection;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.util.ReflectionUtils;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
public class CascadeCallback implements ReflectionUtils.FieldCallback {
private Object source;
private MongoTemplate mongoTemplate;
@Override
public void doWith(@NotNull final Field field) throws IllegalArgumentException, IllegalAccessException {
try {
ReflectionUtils.makeAccessible(field);
if (field.isAnnotationPresent(DBRef.class) && field.isAnnotationPresent(CascadeSave.class)) {
final Object fieldValue = field.get(getSource());
if(Collection.class.isAssignableFrom(field.getType())) {
Collection<?> collection = (Collection<?>) fieldValue;
mongoTemplate.insertAll(collection);
}
else {
if (fieldValue != null) {
//final FieldCallback callback = new FieldCallback();
//ReflectionUtils.doWithFields(fieldValue.getClass(), callback);
mongoTemplate.insert(fieldValue);
}
}
}
} catch (InaccessibleObjectException ignored) {
}
}
}

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/CascadeSave.java
View File

@ -1,12 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
public @interface CascadeSave {
//
}

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/CascadeSaveMongoEventListener.java
View File

@ -1,21 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener;
import org.springframework.data.mongodb.core.mapping.event.BeforeConvertEvent;
import org.springframework.util.ReflectionUtils;
public class CascadeSaveMongoEventListener extends AbstractMongoEventListener<Object> {
@Autowired
private MongoTemplate mongoTemplate;
@Override
public void onBeforeConvert(BeforeConvertEvent<Object> event) {
Object source = event.getSource();
ReflectionUtils.doWithFields(source.getClass(),
new CascadeCallback(source, mongoTemplate));
}
}

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogDocument.java
View File

@ -5,6 +5,8 @@ import java.util.List;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.IndexDirection;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
@ -12,6 +14,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@ -20,20 +23,23 @@ import lombok.Setter;
@Setter
@AllArgsConstructor
@NoArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@Document(collection = "entity-logs")
public class EntityLogDocument {
@Id
@EqualsAndHashCode.Include
private String id;
private String dossierId;
private String fileId;
private long analysisVersion;
//@Indexed(direction = IndexDirection.DESCENDING)
private int analysisNumber;
@DBRef
@CascadeSave
private List<EntityLogEntryDocument> entityLogEntryDocument = new ArrayList<>();
private List<EntityLogLegalBasis> legalBasis = new ArrayList<>();

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogDocumentNotFoundException.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.redaction.v1.server.mongo;
public class EntityLogDocumentNotFoundException extends RuntimeException {
public EntityLogDocumentNotFoundException(String errorMessage) {
super(errorMessage);
}
}

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogEntryDocument.java
View File

@ -6,7 +6,10 @@ import java.util.List;
import java.util.Set;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.CompoundIndex;
import org.springframework.data.mongodb.core.index.CompoundIndexes;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.data.redis.core.index.Indexed;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
@ -26,12 +29,15 @@ import lombok.experimental.FieldDefaults;
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
@Document(collection = "entity-log-entries")
public class EntityLogEntryDocument {
/**@CompoundIndexes({
@CompoundIndex(name = "changes_analysisNumber", def = "{ 'changes.analysisNumber': 1 }")
})**/ public class EntityLogEntryDocument {
@Id
@EqualsAndHashCode.Include
String id;
String entryId;
String entityLogId;
@ -62,10 +68,8 @@ public class EntityLogEntryDocument {
boolean excluded;
@EqualsAndHashCode.Exclude
List<Change> changes = new ArrayList<>();
@EqualsAndHashCode.Exclude
List<ManualChange> manualChanges = new ArrayList<>();
Set<Engine> engines = new HashSet<>();

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogEntryDocumentRepository.java
View File

@ -12,4 +12,10 @@ public interface EntityLogEntryDocumentRepository extends MongoRepository<Entity
@Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : ?1 }")
List<EntityLogEntryDocument> findByEntityLogIdAndChangesAnalysisNumber(String entityLogId, int analysisNumber);
@Query("{ 'entityLogId' : ?0}")
List<EntityLogEntryDocument> findByEntityLogId(String entityLogId);
@Query(value = "{ 'entityLogId' : ?0}", delete = true)
void deleteByEntityLogId(String entityLogId);
}

123
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java
View File

@ -6,6 +6,7 @@ import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -25,9 +26,114 @@ public class EntityLogMongoService {
}
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(new EntityLogDocument(dossierId, fileId, entityLog));
entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
.toList());
}
// this does everything : insert when not found and update if found
// todo: remove and replace when services use insert,update,delete correctly
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogDocumentRepository.save(new EntityLogDocument(dossierId, fileId, entityLog));
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId));
if (optionalEntityLogDocument.isEmpty()) {
// throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId));
insertEntityLog(dossierId, fileId, entityLog);
return;
}
EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get();
List<EntityLogEntryDocument> oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocument();
EntityLogDocument newEntityLogDocument = new EntityLogDocument(dossierId, fileId, entityLog);
List<EntityLogEntryDocument> newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocument();
List<EntityLogEntryDocument> toUpdate = new ArrayList<>(oldEntityLogEntryDocuments);
toUpdate.retainAll(newEntityLogEntryDocuments);
List<EntityLogEntryDocument> toRemove = new ArrayList<>(oldEntityLogEntryDocuments);
toRemove.removeAll(toUpdate);
List<EntityLogEntryDocument> toInsert = new ArrayList<>(newEntityLogEntryDocuments);
toInsert.removeAll(toUpdate);
entityLogEntryDocumentRepository.saveAll(toUpdate);
entityLogEntryDocumentRepository.deleteAll(toRemove);
entityLogEntryDocumentRepository.insert(toInsert);
entityLogDocumentRepository.save(newEntityLogDocument);
}
public void deleteEntityLog(String dossierId, String fileId) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
entityLogDocumentRepository.deleteById(entityLogId);
entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId);
}
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocument().addAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.insert(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList());
}
public void deleteEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocument().removeAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
@NotNull
private EntityLogDocument getEntityLogDocument(String entityLogId) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId);
if (optionalEntityLogDocument.isEmpty()) {
throw new EntityLogDocumentNotFoundException(String.format("Entity log not found for %s", entityLogId));
}
return optionalEntityLogDocument.get();
}
@ -38,9 +144,9 @@ public class EntityLogMongoService {
}
public boolean entityLogDocumentExists(String id) {
public boolean entityLogDocumentExists(String dossierId, String fileId) {
return entityLogDocumentRepository.existsById(id);
return entityLogDocumentRepository.existsById(EntityLogDocument.getDocumentId(dossierId, fileId));
}
@ -58,6 +164,8 @@ public class EntityLogMongoService {
.map(EntityLogMongoService::fromDocument)
.toList();
}
public List<EntityLogEntry> findAllEntityLogEntriesByAnalysisNumber(String dossierId, String fileId, int analysisNumber) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumber(EntityLogDocument.getDocumentId(dossierId, fileId), analysisNumber)
@ -67,6 +175,15 @@ public class EntityLogMongoService {
}
public List<EntityLogEntry> findAllEntriesByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogId(EntityLogDocument.getDocumentId(dossierId, fileId))
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
private static EntityLog fromDocument(EntityLogDocument entityLogDocument) {
EntityLog entityLog = new EntityLog();

35
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/FieldCallback.java
View File

@ -1,35 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.lang.reflect.Field;
import java.lang.reflect.InaccessibleObjectException;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.annotation.Id;
import org.springframework.util.ReflectionUtils;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FieldCallback implements ReflectionUtils.FieldCallback {
private boolean idFound;
@Override
public void doWith(@NotNull final Field field) throws IllegalArgumentException {
try {
ReflectionUtils.makeAccessible(field);
if (field.isAnnotationPresent(Id.class)) {
idFound = true;
}
} catch (InaccessibleObjectException ignored) {
}
}
}

38
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongodeprecated/EntityLogDocumentService.java
View File

@ -1,38 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongodeprecated;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentRepository;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
public class EntityLogDocumentService {
private final EntityLogDocumentRepository entityLogDocumentRepository;
public EntityLogDocumentService(EntityLogDocumentRepository entityLogDocumentRepository) {this.entityLogDocumentRepository = entityLogDocumentRepository;}
public EntityLogDocument saveEntityLogDocument(EntityLogDocument entityLogDocument) {
return entityLogDocumentRepository.save(entityLogDocument);
}
public Optional<EntityLogDocument> findEntityLogDocumentById(String id) {
return entityLogDocumentRepository.findById(id);
}
public boolean entityLogDocumentExists(String id) {
return entityLogDocumentRepository.existsById(id);
}
}

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongodeprecated/_EntityLogDocumentRepository.java
View File

@ -1,26 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongodeprecated;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Repository
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@RequiredArgsConstructor
public class _EntityLogDocumentRepository {
MongoTemplate mongoTemplate;
public EntityLogDocument saveEntityLogDocument(EntityLogDocument entityLogDocument) {
return mongoTemplate.save(entityLogDocument);
}
public EntityLogDocument findEntityLogDocumentById(String id) {
return mongoTemplate.findById(id, EntityLogDocument.class);
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -211,8 +211,7 @@ public class RedactionStorageService {
public boolean entityLogExists(String dossierId, String fileId) {
//return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG));
return entityLogMongoService.entityLogDocumentExists(fileId);
return entityLogMongoService.entityLogDocumentExists(dossierId, fileId);
}

1
redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml
View File

@ -47,6 +47,7 @@ spring:
password: ${REDIS_PASSWORD}
timeout: 60000
mongodb:
auto-index-creation: true
# todo: multi-tenancy
database: redaction
host: ${MONGODB_HOST:localhost}

172
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java
View File

@ -1,11 +1,14 @@
package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
@ -31,6 +34,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
@ -80,6 +85,7 @@ public class MyOtherMongoTest {
private static final String TEST_DOSSIER_2_ID = "bigentestdossierid";
private static final String TEST_FILE_2_ID = "bigentityfileid";
private static final String TEST_FILE_3_ID = "upd_bigentityfileid";
private static final String TEST_DOSSIER_ID = "testdossierid";
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
public static final String COLLECTION_ENTITY_LOG = "entity-logs";
@ -110,7 +116,6 @@ public class MyOtherMongoTest {
TenantContext.setTenantId("redaction");
when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(MongoDBConnection.builder()
.host("localhost")
.port("27017")
@ -120,19 +125,70 @@ public class MyOtherMongoTest {
.build());
}
@Test
@SneakyThrows
public void testUpdateBigEntityLogDocumentByAddingOne() {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLogEntry entityLogEntry = objectMapper.readValue(NEW_ENTITY_LOG_ENTRY, EntityLogEntry.class);
entityLogMongoService.insertEntityLogEntries(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, List.of(entityLogEntry));
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getEntityLogEntry().size(), 1707);
}
@Test
@SneakyThrows
public void testUpdateBigEntityLogDocumentByNewLog() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_3_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLog.setAnalysisNumber(entityLog.getAnalysisNumber()+1);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getAnalysisNumber(), entityLog.getAnalysisNumber());
assertEquals(found.get().getEntityLogEntry().size(), 1707);
}
@Test
@SneakyThrows
public void testUpdateBigEntityLogDocumentAnalysisNumber() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_2_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLog.setAnalysisNumber(entityLog.getAnalysisNumber()+1);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getAnalysisNumber(), entityLog.getAnalysisNumber());
}
@Test
@SneakyThrows
public void testFindEntries() {
var latest = entityLogMongoService.findLatestAnalysisNumber(TEST_DOSSIER_2_ID, TEST_FILE_2_ID)
.orElseThrow();
var latestChangedEntries = entityLogMongoService.findAllEntityLogEntriesByAnalysisNumber(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, latest);
assertEquals(latestChangedEntries.size(), 491);
assertEquals(latestChangedEntries.size(), 490);
var manuallyChangedEntries = entityLogMongoService.findAllEntityLogEntriesWithManualChanges(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertEquals(manuallyChangedEntries.size(), 1014);
assertEquals(manuallyChangedEntries.size(), 1013);
}
@ -149,7 +205,18 @@ public class MyOtherMongoTest {
@Test
@SneakyThrows
public void testSaveBigEntityLogDocument() {
public void testDeleteEntityLog() {
entityLogMongoService.deleteEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertFalse(entityLogMongoService.entityLogDocumentExists(TEST_DOSSIER_2_ID, TEST_FILE_2_ID));
assertEquals(entityLogMongoService.findAllEntriesByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID).size(), 0);
}
@Test
@SneakyThrows
public void testInsertBigEntityLogDocument() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_2_ID));
ObjectMapper objectMapper = new ObjectMapper();
@ -157,16 +224,16 @@ public class MyOtherMongoTest {
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assert (found.isPresent());
assertTrue(found.isPresent());
}
@Test
@SneakyThrows
public void testSaveEntityLogDocument() {
public void testInsertEntityLogDocument() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
ObjectMapper objectMapper = new ObjectMapper();
@ -174,7 +241,7 @@ public class MyOtherMongoTest {
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
}
@ -279,7 +346,8 @@ public class MyOtherMongoTest {
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
TestPropertyValues.of("MONGODB_HOST=" + HOSTNAME,
TestPropertyValues.of("spring.data.mongodb.auto-index-creation=true",
"MONGODB_HOST=" + HOSTNAME,
"MONGODB_PORT=" + PORT,
"MONGODB_USER=" + USERNAME,
"MONGODB_PASSWORD=" + PASSWORD,
@ -292,4 +360,88 @@ public class MyOtherMongoTest {
}
private final String NEW_ENTITY_LOG_ENTRY = """
{
"id": "05240998f2e657d739d59d220094702a",
"type": "CBI_author",
"entryType": "ENTITY",
"state": "REMOVED",
"value": "Amato",
"reason": "Author found",
"matchedRule": "CBI.0.0",
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
"imported": false,
"containingNodeId": [
1,
0
],
"closestHeadline": "",
"section": "[1, 0]: Paragraph: Almond R.H. Almond Richard",
"color": null,
"positions": [
{
"rectangle": [
56.8,
639.3,
32.59199,
12.642
],
"pageNumber": 2
}
],
"textBefore": "S. Amarasekare Amarasingham ",
"textAfter": " Amato S.L Amato",
"startOffset": 5137,
"endOffset": 5142,
"imageHasTransparency": false,
"dictionaryEntry": true,
"dossierDictionaryEntry": false,
"excluded": false,
"changes": [
{
"analysisNumber": 1,
"type": "ADDED",
"dateTime": "2024-03-13T08:44:54.811245839Z"
},
{
"analysisNumber": 3,
"type": "REMOVED",
"dateTime": "2024-03-13T08:47:11.339124572Z"
},
{
"analysisNumber": 4,
"type": "REMOVED",
"dateTime": "2024-03-13T08:47:19.694336707Z"
},
{
"analysisNumber": 5,
"type": "REMOVED",
"dateTime": "2024-03-13T08:48:21.670287664Z"
},
{
"analysisNumber": 6,
"type": "REMOVED",
"dateTime": "2024-03-13T08:58:41.082712591Z"
},
{
"analysisNumber": 7,
"type": "REMOVED",
"dateTime": "2024-03-13T08:59:31.444615299Z"
},
{
"analysisNumber": 8,
"type": "REMOVED",
"dateTime": "2024-03-13T09:05:22.006293189Z"
}
],
"manualChanges": [],
"engines": [
"DICTIONARY"
],
"reference": [],
"importedRedactionIntersections": [],
"numberOfComments": 0
}
""";
}

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml
View File

@ -14,6 +14,7 @@ spring:
type: NONE
data:
mongodb:
auto-index-creation: true
# todo: multi-tenancy
database: redaction
host: ${MONGODB_HOST:localhost}

81
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/bigentityfileid.ENTITY_LOG.json
View File

@ -120271,87 +120271,6 @@
"reference": [],
"importedRedactionIntersections": [],
"numberOfComments": 0
},
{
"id": "05240998f2e657d739d59d220094702a",
"type": "CBI_author",
"entryType": "ENTITY",
"state": "REMOVED",
"value": "Amato",
"reason": "Author found",
"matchedRule": "CBI.0.0",
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
"imported": false,
"containingNodeId": [
1,
0
],
"closestHeadline": "",
"section": "[1, 0]: Paragraph: Almond R.H. Almond Richard",
"color": null,
"positions": [
{
"rectangle": [
56.8,
639.3,
32.59199,
12.642
],
"pageNumber": 2
}
],
"textBefore": "S. Amarasekare Amarasingham ",
"textAfter": " Amato S.L Amato",
"startOffset": 5137,
"endOffset": 5142,
"imageHasTransparency": false,
"dictionaryEntry": true,
"dossierDictionaryEntry": false,
"excluded": false,
"changes": [
{
"analysisNumber": 1,
"type": "ADDED",
"dateTime": "2024-03-13T08:44:54.811245839Z"
},
{
"analysisNumber": 3,
"type": "REMOVED",
"dateTime": "2024-03-13T08:47:11.339124572Z"
},
{
"analysisNumber": 4,
"type": "REMOVED",
"dateTime": "2024-03-13T08:47:19.694336707Z"
},
{
"analysisNumber": 5,
"type": "REMOVED",
"dateTime": "2024-03-13T08:48:21.670287664Z"
},
{
"analysisNumber": 6,
"type": "REMOVED",
"dateTime": "2024-03-13T08:58:41.082712591Z"
},
{
"analysisNumber": 7,
"type": "REMOVED",
"dateTime": "2024-03-13T08:59:31.444615299Z"
},
{
"analysisNumber": 8,
"type": "REMOVED",
"dateTime": "2024-03-13T09:05:22.006293189Z"
}
],
"manualChanges": [],
"engines": [
"DICTIONARY"
],
"reference": [],
"importedRedactionIntersections": [],
"numberOfComments": 0
}
],
"legalBasis": [

120408
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/upd_bigentityfileid.ENTITY_LOG.json Normal file
View File

File diff suppressed because it is too large Load Diff