Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* added mongodb documents, repositories and EntityLogMongoService
* refactored places where entity logs were stored and loaded to use new logic
This commit is contained in:
maverickstuder 2024-03-18 15:05:41 +01:00
parent f8741b54d3
commit cfdd6a6924
15 changed files with 528 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java
View File

@ -36,7 +36,7 @@ public class AdminInterfaceController {
public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
@ -141,7 +141,7 @@ public class AdminInterfaceController {
var fileId = file.getId();
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);

5
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -34,8 +34,8 @@ dependencies {
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight:jobs-commons:0.10.0")
api("com.knecon.fforesight:database-tenant-commons:0.21.0")
api("com.knecon.fforesight:keycloak-commons:0.25.0")
api("com.knecon.fforesight:database-tenant-commons:maverick-mongo")
api("com.knecon.fforesight:keycloak-commons:maverick-mongo")
api("com.knecon.fforesight:tracing-commons:0.5.0")
api("com.knecon.fforesight:swagger-commons:0.5.0")
api("com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:0.4.0")
@ -43,6 +43,7 @@ dependencies {
api("org.springframework.boot:spring-boot-starter-mail:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-jpa:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-web:${springBootStarterVersion}")
api("com.iqser.red.commons:spring-commons:2.1.0")

75
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogDocument.java Normal file
View File

@ -0,0 +1,75 @@
package com.iqser.red.service.persistence.management.v1.processor.document;
import java.util.ArrayList;
import java.util.List;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@Document(collection = "entity-logs")
public class EntityLogDocument {
@Id
@EqualsAndHashCode.Include
private String id;
private String dossierId;
private String fileId;
private long analysisVersion;
//@Indexed(direction = IndexDirection.DESCENDING)
private int analysisNumber;
@DBRef
private List<EntityLogEntryDocument> entityLogEntryDocument = new ArrayList<>();
private List<EntityLogLegalBasis> legalBasis = new ArrayList<>();
private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1;
public EntityLogDocument(String dossierId, String fileId, EntityLog entityLog) {
this.id = getDocumentId(dossierId, fileId);
this.dossierId = dossierId;
this.fileId = fileId;
this.analysisVersion = entityLog.getAnalysisVersion();
this.analysisNumber = entityLog.getAnalysisNumber();
this.entityLogEntryDocument = new ArrayList<>(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(this.id, entityLogEntry)).toList());
this.legalBasis = entityLog.getLegalBasis();
this.dictionaryVersion = entityLog.getDictionaryVersion();
this.dossierDictionaryVersion = entityLog.getDossierDictionaryVersion();
this.rulesVersion = entityLog.getRulesVersion();
this.legalBasisVersion = entityLog.getLegalBasisVersion();
}
public static String getDocumentId(String dossierId, String fileId) {
return dossierId + "/" + fileId;
}
}

114
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogEntryDocument.java Normal file
View File

@ -0,0 +1,114 @@
package com.iqser.red.service.persistence.management.v1.processor.document;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
@Document(collection = "entity-log-entries")
//@CompoundIndexes({
//@CompoundIndex(name = "changes_analysisNumber", def = "{ 'changes.analysisNumber': 1 }")
// })
public class EntityLogEntryDocument {
@Id
@EqualsAndHashCode.Include
String id;
String entryId;
String entityLogId;
String type;
EntryType entryType;
EntryState state;
String value;
String reason;
String matchedRule;
String legalBasis;
List<Integer> containingNodeId;
String closestHeadline;
String section;
List<Position> positions = new ArrayList<>();
String textBefore;
String textAfter;
int startOffset;
int endOffset;
boolean imageHasTransparency;
boolean dictionaryEntry;
boolean dossierDictionaryEntry;
boolean excluded;
List<Change> changes = new ArrayList<>();
List<ManualChange> manualChanges = new ArrayList<>();
Set<Engine> engines = new HashSet<>();
Set<String> reference = new HashSet<>();
Set<String> importedRedactionIntersections = new HashSet<>();
int numberOfComments;
public EntityLogEntryDocument(String entityLogId, EntityLogEntry entityLogEntry) {
this.id = entityLogId + "/" + entityLogEntry.getId();
this.entryId = entityLogEntry.getId();
this.entityLogId = entityLogId;
this.type = entityLogEntry.getType();
this.entryType = entityLogEntry.getEntryType();
this.state = entityLogEntry.getState();
this.value = entityLogEntry.getValue();
this.reason = entityLogEntry.getReason();
this.matchedRule = entityLogEntry.getMatchedRule();
this.legalBasis = entityLogEntry.getLegalBasis();
this.containingNodeId = new ArrayList<>(entityLogEntry.getContainingNodeId());
this.closestHeadline = entityLogEntry.getClosestHeadline();
this.section = entityLogEntry.getSection();
this.positions = new ArrayList<>(entityLogEntry.getPositions());
this.textBefore = entityLogEntry.getTextBefore();
this.textAfter = entityLogEntry.getTextAfter();
this.startOffset = entityLogEntry.getStartOffset();
this.endOffset = entityLogEntry.getEndOffset();
this.imageHasTransparency = entityLogEntry.isImageHasTransparency();
this.dictionaryEntry = entityLogEntry.isDictionaryEntry();
this.dossierDictionaryEntry = entityLogEntry.isDossierDictionaryEntry();
this.excluded = entityLogEntry.isExcluded();
this.changes = new ArrayList<>(entityLogEntry.getChanges());
this.manualChanges = new ArrayList<>(entityLogEntry.getManualChanges());
this.engines = new HashSet<>(entityLogEntry.getEngines());
this.reference = new HashSet<>(entityLogEntry.getReference());
this.importedRedactionIntersections = new HashSet<>(entityLogEntry.getImportedRedactionIntersections());
this.numberOfComments = entityLogEntry.getNumberOfComments();
}
}

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/exception/EntityLogDocumentNotFoundException.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.exception;
public class EntityLogDocumentNotFoundException extends RuntimeException {
public EntityLogDocumentNotFoundException(String errorMessage) {
super(errorMessage);
}
}

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -200,6 +200,7 @@ public class SaasMigrationService implements TenantSyncService {
}
//todo: 8702
private boolean entityLogMigrationFilesExist(String dossierId, String fileId) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists(

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeRenameMigration15.java
View File

@ -69,7 +69,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration {
continue;
}
if (!fileManagementStorageService.objectExists(file.getDossierId(), file.getId(), FileType.ENTITY_LOG)) {
if (!fileManagementStorageService.entityLogExists(file.getDossierId(), file.getId())) {
continue;
}
@ -92,7 +92,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration {
}
}
}
fileManagementStorageService.storeJSONObject(file.getDossierId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(file.getDossierId(), file.getId(), entityLog);
}
}

30
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntityLogMongoService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -36,6 +37,8 @@ public class FileManagementStorageService {
private final StorageService storageService;
private final EntityLogMongoService entityLogMongoService;
@SneakyThrows
public byte[] getStoredObjectBytes(String dossierId, String fileId, FileType fileType) {
@ -99,14 +102,20 @@ public class FileManagementStorageService {
public EntityLog getEntityLog(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("EntityLog does not exist");
throw new NotFoundException(String.format("EntityLog does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId));
} catch (StorageException e) {
throw new InternalServerErrorException(e.getMessage());
}
return entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId)
.orElseThrow(() -> new NotFoundException(String.format("EntityLog does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId)));
}
@SneakyThrows
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog);
}
public boolean entityLogExists(String dossierId, String fileId) {
return entityLogMongoService.entityLogDocumentExists(dossierId, fileId);
}
@ -161,6 +170,11 @@ public class FileManagementStorageService {
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType));
}
public void deleteEntityLog(String dossierId, String fileId) {
entityLogMongoService.deleteEntityLog(dossierId, fileId);
}
public void deleteObject(String storageId) {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -690,7 +690,7 @@ public class FileStatusService {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ORIGIN);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
@ -719,7 +719,7 @@ public class FileStatusService {
OffsetDateTime now = OffsetDateTime.now();
// remove everything related to redaction
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
// wipe comments

238
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntityLogMongoService.java Normal file
View File

@ -0,0 +1,238 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.management.v1.processor.exception.EntityLogDocumentNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@Service
public class EntityLogMongoService {
private final EntityLogDocumentRepository entityLogDocumentRepository;
private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
public EntityLogMongoService(EntityLogDocumentRepository entityLogDocumentRepository, EntityLogEntryDocumentRepository entityLogEntryDocumentRepository) {
this.entityLogDocumentRepository = entityLogDocumentRepository;
this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;
}
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(new EntityLogDocument(dossierId, fileId, entityLog));
entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
.toList());
}
// this does everything : insert when not found and update if found
// todo: remove and replace when services use insert,update,delete correctly
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId));
if (optionalEntityLogDocument.isEmpty()) {
// throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId));
insertEntityLog(dossierId, fileId, entityLog);
return;
}
EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get();
List<EntityLogEntryDocument> oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocument();
EntityLogDocument newEntityLogDocument = new EntityLogDocument(dossierId, fileId, entityLog);
List<EntityLogEntryDocument> newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocument();
List<EntityLogEntryDocument> toUpdate = new ArrayList<>(oldEntityLogEntryDocuments);
toUpdate.retainAll(newEntityLogEntryDocuments);
List<EntityLogEntryDocument> toRemove = new ArrayList<>(oldEntityLogEntryDocuments);
toRemove.removeAll(toUpdate);
List<EntityLogEntryDocument> toInsert = new ArrayList<>(newEntityLogEntryDocuments);
toInsert.removeAll(toUpdate);
entityLogEntryDocumentRepository.saveAll(toUpdate);
entityLogEntryDocumentRepository.deleteAll(toRemove);
entityLogEntryDocumentRepository.insert(toInsert);
entityLogDocumentRepository.save(newEntityLogDocument);
}
public void deleteEntityLog(String dossierId, String fileId) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
entityLogDocumentRepository.deleteById(entityLogId);
entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId);
}
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocument().addAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.insert(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList());
}
public void deleteEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocument().removeAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
private EntityLogDocument getEntityLogDocument(String entityLogId) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId);
if (optionalEntityLogDocument.isEmpty()) {
throw new EntityLogDocumentNotFoundException(String.format("Entity log not found for %s", entityLogId));
}
return optionalEntityLogDocument.get();
}
public Optional<EntityLog> findEntityLogByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId))
.map(EntityLogMongoService::fromDocument);
}
public boolean entityLogDocumentExists(String dossierId, String fileId) {
return entityLogDocumentRepository.existsById(EntityLogDocument.getDocumentId(dossierId, fileId));
}
public Optional<Integer> findLatestAnalysisNumber(String dossierId, String fileId) {
return entityLogDocumentRepository.findAnalysisNumberById(EntityLogDocument.getDocumentId(dossierId, fileId))
.map(EntityLogDocument::getAnalysisNumber);
}
public List<EntityLogEntry> findAllEntityLogEntriesWithManualChanges(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndManualChangesNotEmpty(EntityLogDocument.getDocumentId(dossierId, fileId))
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
public List<EntityLogEntry> findAllEntityLogEntriesByAnalysisNumber(String dossierId, String fileId, int analysisNumber) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumber(EntityLogDocument.getDocumentId(dossierId, fileId), analysisNumber)
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
public List<EntityLogEntry> findAllEntriesByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogId(EntityLogDocument.getDocumentId(dossierId, fileId))
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
private static EntityLog fromDocument(EntityLogDocument entityLogDocument) {
EntityLog entityLog = new EntityLog();
entityLog.setAnalysisVersion(entityLogDocument.getAnalysisVersion());
entityLog.setAnalysisNumber(entityLogDocument.getAnalysisNumber());
entityLog.setEntityLogEntry(entityLogDocument.getEntityLogEntryDocument()
.stream()
.map(EntityLogMongoService::fromDocument)
.collect(Collectors.toList()));
entityLog.setLegalBasis(entityLogDocument.getLegalBasis());
entityLog.setDictionaryVersion(entityLogDocument.getDictionaryVersion());
entityLog.setDossierDictionaryVersion(entityLogDocument.getDossierDictionaryVersion());
entityLog.setRulesVersion(entityLogDocument.getRulesVersion());
entityLog.setLegalBasisVersion(entityLogDocument.getLegalBasisVersion());
return entityLog;
}
private static EntityLogEntry fromDocument(EntityLogEntryDocument entityLogEntryDocument) {
EntityLogEntry entityLogEntry = new EntityLogEntry();
entityLogEntry.setId(entityLogEntryDocument.getEntryId());
entityLogEntry.setState(entityLogEntryDocument.getState());
entityLogEntry.setValue(entityLogEntryDocument.getValue());
entityLogEntry.setReason(entityLogEntryDocument.getReason());
entityLogEntry.setMatchedRule(entityLogEntryDocument.getMatchedRule());
entityLogEntry.setLegalBasis(entityLogEntryDocument.getLegalBasis());
entityLogEntry.setContainingNodeId(new ArrayList<>(entityLogEntryDocument.getContainingNodeId()));
entityLogEntry.setClosestHeadline(entityLogEntryDocument.getClosestHeadline());
entityLogEntry.setSection(entityLogEntryDocument.getSection());
entityLogEntry.setPositions(new ArrayList<>(entityLogEntryDocument.getPositions()));
entityLogEntry.setTextBefore(entityLogEntryDocument.getTextBefore());
entityLogEntry.setTextAfter(entityLogEntryDocument.getTextAfter());
entityLogEntry.setStartOffset(entityLogEntryDocument.getStartOffset());
entityLogEntry.setEndOffset(entityLogEntryDocument.getEndOffset());
entityLogEntry.setImageHasTransparency(entityLogEntryDocument.isImageHasTransparency());
entityLogEntry.setDictionaryEntry(entityLogEntryDocument.isDictionaryEntry());
entityLogEntry.setDossierDictionaryEntry(entityLogEntryDocument.isDossierDictionaryEntry());
entityLogEntry.setExcluded(entityLogEntryDocument.isExcluded());
entityLogEntry.setChanges(new ArrayList<>(entityLogEntryDocument.getChanges()));
entityLogEntry.setManualChanges(new ArrayList<>(entityLogEntryDocument.getManualChanges()));
entityLogEntry.setEngines(new HashSet<>(entityLogEntryDocument.getEngines()));
entityLogEntry.setReference(new HashSet<>(entityLogEntryDocument.getReference()));
entityLogEntry.setImportedRedactionIntersections(new HashSet<>(entityLogEntryDocument.getImportedRedactionIntersections()));
entityLogEntry.setNumberOfComments(entityLogEntryDocument.getNumberOfComments());
return entityLogEntry;
}
}

15
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogDocumentRepository.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.util.Optional;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.data.repository.query.Param;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {
@Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }")
Optional<EntityLogDocument> findAnalysisNumberById(@Param("_id") String id);
}

23
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogEntryDocumentRepository.java Normal file
View File

@ -0,0 +1,23 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.util.List;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
public interface EntityLogEntryDocumentRepository extends MongoRepository<EntityLogEntryDocument, String> {
@Query("{ 'entityLogId' : ?0, 'manualChanges' : { $exists: true, $not: { $size: 0 } } }")
List<EntityLogEntryDocument> findByEntityLogIdAndManualChangesNotEmpty(String entityLogId);
@Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : ?1 }")
List<EntityLogEntryDocument> findByEntityLogIdAndChangesAnalysisNumber(String entityLogId, int analysisNumber);
@Query("{ 'entityLogId' : ?0}")
List<EntityLogEntryDocument> findByEntityLogId(String entityLogId);
@Query(value = "{ 'entityLogId' : ?0}", delete = true)
void deleteByEntityLogId(String entityLogId);
}

3
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java
View File

@ -64,9 +64,8 @@ public class FileTesterAndProvider {
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isGreaterThanOrEqualTo(1);
fileManagementStorageService.storeJSONObject(dossier.getId(),
fileManagementStorageService.saveEntityLog(dossier.getId(),
file.getFileId(),
FileType.ENTITY_LOG,
new EntityLog(1,
1,
List.of(EntityLogEntry.builder()

40
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -139,7 +139,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyBoolean())).thenReturn(entityLog);
@ -333,7 +333,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -393,7 +393,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -448,7 +448,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -543,7 +543,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1);
fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file1.getDossierId()), eq(file1.getFileId()), any(), anyBoolean())).thenReturn(entityLog1);
@ -562,7 +562,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2);
fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2);
@ -708,7 +708,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1);
fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file1.getDossierId()), eq(file1.getFileId()), any(), anyBoolean())).thenReturn(entityLog1);
@ -727,7 +727,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2);
fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2);
@ -877,7 +877,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1);
fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(file1.getDossierId(), file1.getFileId())).thenReturn(entityLog1);
@ -896,7 +896,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2);
fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2);
@ -1043,7 +1043,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1);
fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(file1.getDossierId(), file1.getFileId())).thenReturn(entityLog1);
@ -1062,7 +1062,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2);
fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2);
@ -1186,7 +1186,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -1270,7 +1270,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -1449,7 +1449,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -1648,7 +1648,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -1754,7 +1754,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any())).thenReturn(entityLog);
@ -1864,7 +1864,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any())).thenReturn(entityLog);
@ -1948,8 +1948,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
var recatModel = RecategorizationRequestModel.builder()

1
persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts
View File

@ -11,7 +11,6 @@ dependencies {
api("org.springframework.boot:spring-boot-starter-validation:3.1.3")
api("com.iqser.red.commons:jackson-commons:2.1.0")
api("com.iqser.red.commons:dictionary-merge-commons:1.3.0")
implementation("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}")
testImplementation("com.iqser.red.commons:test-commons:2.1.0")
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
compileOnly("org.springdoc:springdoc-openapi-ui:1.7.0")