Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-9123' into 'master'

Browse Source 
RED-9123: Improve performance of re-analysis (Spike)

Closes RED-9123

See merge request redactmanager/persistence-service!771
This commit is contained in:
Maverick Studer 2024-10-07 13:31:16 +02:00
commit 7d52b4b0a0
32 changed files with 64643 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java
View File

@ -35,16 +35,7 @@ public class AdminInterfaceController {
@PostMapping("/reset-file")
public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.AZURE_NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.FIGURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TABLES);
fileManagementStorageService.deleteAllObjects(dossierId, fileId);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
@ -143,12 +134,7 @@ public class AdminInterfaceController {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.AZURE_NER_ENTITIES);
fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
}

9
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -30,7 +30,7 @@ dependencies {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight:layoutparser-service-internal-api:0.163.0") {
api("com.knecon.fforesight:layoutparser-service-internal-api:0.181.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
@ -44,7 +44,10 @@ dependencies {
}
implementation("com.knecon.fforesight:llm-service-api:1.17.0")
api("com.knecon.fforesight:jobs-commons:0.10.0")
api("com.knecon.fforesight:tenant-commons:0.30.0")
api("com.iqser.red.commons:storage-commons:2.50.0")
api("com.knecon.fforesight:tenant-commons:0.30.0") {
exclude(group = "com.iqser.red.commons", module = "storage-commons")
}
api("com.knecon.fforesight:database-tenant-commons:0.24.0") {
exclude(group = "com.knecon.fforesight", module = "tenant-commons")
}
@ -64,7 +67,6 @@ dependencies {
api("org.springframework.security:spring-security-messaging:6.1.3")
api("com.iqser.red.commons:spring-commons:2.1.0")
api("com.iqser.red.commons:jackson-commons:2.3.0")
api("com.iqser.red.commons:storage-commons:2.49.0")
api("com.iqser.red.commons:spring-boot-starter-web-custom-commons:2.1.0")
api("com.iqser.red.commons:metric-commons:2.3.0")
api("org.apache.commons:commons-compress:1.21")
@ -75,6 +77,7 @@ dependencies {
api("commons-validator:commons-validator:1.7")
api("com.opencsv:opencsv:5.9")
implementation("com.google.protobuf:protobuf-java:4.27.1")
implementation("org.mapstruct:mapstruct:1.5.5.Final")
annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final")

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java
View File

@ -207,6 +207,9 @@ public class FileEntity {
@Column
private Integer overwriteFileCounter;
@Column
private boolean protobufMigrationDone;
public OffsetDateTime getLastOCRTime() {
return this.ocrStartTime;

27
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/jobs/CreateJobsConfiguration.java
View File

@ -75,6 +75,7 @@ public class CreateJobsConfiguration {
.build();
}
@Bean
public Trigger deletedFilesCleanupJobTrigger() throws ParseException {
@ -86,6 +87,19 @@ public class CreateJobsConfiguration {
.build();
}
@Bean
public Trigger documentDataFilesConversionJobTrigger() throws ParseException {
return TriggerBuilder.newTrigger()
.forJob(documentDataFilesConversionJobDetail())
.withIdentity("DocumentDataFilesConversionJobTrigger")
.withDescription("Triggers DocumentDataFilesConversionJob nightly at 2 AM and runs for 2 hours")
.withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 0 2 * * ?")))
.build();
}
@Bean
public JobDetail softDeletedFilesCleanupJobDetail() {
@ -97,6 +111,7 @@ public class CreateJobsConfiguration {
.build();
}
@Bean
public JobDetail deletedFilesCleanupJobDetail() {
@ -109,6 +124,18 @@ public class CreateJobsConfiguration {
}
@Bean
public JobDetail documentDataFilesConversionJobDetail() {
return JobBuilder.newJob()
.ofType(DocumentDataFilesConversionJob.class)
.storeDurably()
.withIdentity("DocumentDataFilesConversionJob")
.withDescription("Convert document data files from json to protobuf format")
.build();
}
@Bean
public Trigger keyCloakUserSyncJobTrigger() throws ParseException {

107
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/QueueRenameMigration23.java
View File

@ -1,107 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
//import java.util.List;
//
//import org.springframework.amqp.core.AmqpAdmin;
//import org.springframework.stereotype.Service;
//
//import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
//
//import lombok.Setter;
//import lombok.extern.slf4j.Slf4j;
//
//
//@Slf4j
//@Setter
//@Service
//public class QueueRenameMigration23 extends Migration {
//
// private final AmqpAdmin amqpAdmin;
//
// private static final String NAME = "Migration for renaming of most queues";
// private static final long VERSION = 23;
// private static final List<String> queueNames = List.of("analysis_flag_calculation_queue",
// "cv_analysis_dead_letter_queue",
// "cv_analysis_request_queue",
// "cv_analysis_response_queue",
// "deleteFromIndexDLQ",
// "deleteFromIndexQueue",
// "downloadDLQ",
// "downloadQueue",
// "download_compression_dlq",
// "download_compression_queue",
// "entity_dead_letter_queue",
// "entity_request_queue",
// "entity_response_queue",
// "exportDownloadDLQ",
// "exportDownloadQueue",
// "image_dead_letter_queue",
// "image_request_queue",
// "image_response_queue",
// "indexingDQL",
// "indexingQueue",
// "layout_parsing_dead_letter_queue",
// "layout_parsing_request_queue",
// "layout_parsing_response_queue",
// "migrationDLQ",
// "migrationQueue",
// "migrationResponseQueue",
// "mongo-tenant-created",
// "mongo-tenant-created-dlq",
// "ocr_dead_letter_queue",
// "ocr_request_queue",
// "ocr_response_queue",
// "ocr_status_update_dead_letter_queue",
// "ocr_status_update_response_queue",
// "pdftron_dlq",
// "pdftron_queue",
// "pdftron_result_queue",
// "persistence-service-user-created-queue",
// "persistence-service-user-deleted-queue",
// "persistence-service-user-events-dql",
// "persistence-service-user-own-profile-updated-queue",
// "persistence-service-user-roles-updated-queue",
// "persistence-service-user-status-changed-queue",
// "persistence-service-user-updated-queue",
// "preprocessingDLQ",
// "preprocessingQueue",
// "redactionAnalysisResponseQueue",
// "redactionDQL",
// "redactionPriorityQueue",
// "redactionQueue",
// "reportDLQ",
// "reportQueue",
// "reportResultDLQ",
// "reportResultQueue",
// "tenant-created",
// "tenant-created-dlq",
// "tenant-delete-dlq",
// "tenant-delete-queue",
// "tenant-updated-queue",
// "tenant-updated-dlq",
// "tenant-sync",
// "tenant-sync-dlq",
// "visual_layout_parsing_service_dead_letter_queue",
// "visual_layout_parsing_service_queue",
// "visual_layout_parsing_service_response_queue");
//
//
// public QueueRenameMigration23(AmqpAdmin amqpAdmin) {
//
// super(NAME, VERSION);
// this.amqpAdmin = amqpAdmin;
// }
//
//
// @Override
// protected void migrate() {
//
// log.info("Migration: Deleting queues...");
// for (String queueName : queueNames) {
// log.info("Migration: Deleting queue {}", queueName);
// amqpAdmin.deleteQueue(queueName);
// }
//
// }
//
//}

107
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/QueueRenameMigration27.java Normal file
View File

@ -0,0 +1,107 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import java.util.List;
import org.springframework.amqp.core.AmqpAdmin;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class QueueRenameMigration27 extends Migration {
private final AmqpAdmin amqpAdmin;
private static final String NAME = "Migration for renaming of most queues";
private static final long VERSION = 27;
private static final List<String> queueNames = List.of("analysis_flag_calculation_queue",
"cv_analysis_dead_letter_queue",
"cv_analysis_request_queue",
"cv_analysis_response_queue",
"deleteFromIndexDLQ",
"deleteFromIndexQueue",
"downloadDLQ",
"downloadQueue",
"download_compression_dlq",
"download_compression_queue",
"entity_dead_letter_queue",
"entity_request_queue",
"entity_response_queue",
"exportDownloadDLQ",
"exportDownloadQueue",
"image_dead_letter_queue",
"image_request_queue",
"image_response_queue",
"indexingDQL",
"indexingQueue",
"layout_parsing_dead_letter_queue",
"layout_parsing_request_queue",
"layout_parsing_response_queue",
"migrationDLQ",
"migrationQueue",
"migrationResponseQueue",
"mongo-tenant-created",
"mongo-tenant-created-dlq",
"ocr_dead_letter_queue",
"ocr_request_queue",
"ocr_response_queue",
"ocr_status_update_dead_letter_queue",
"ocr_status_update_response_queue",
"pdftron_dlq",
"pdftron_queue",
"pdftron_result_queue",
"persistence-service-user-created-queue",
"persistence-service-user-deleted-queue",
"persistence-service-user-events-dql",
"persistence-service-user-own-profile-updated-queue",
"persistence-service-user-roles-updated-queue",
"persistence-service-user-status-changed-queue",
"persistence-service-user-updated-queue",
"preprocessingDLQ",
"preprocessingQueue",
"redactionAnalysisResponseQueue",
"redactionDQL",
"redactionPriorityQueue",
"redactionQueue",
"reportDLQ",
"reportQueue",
"reportResultDLQ",
"reportResultQueue",
"tenant-created",
"tenant-created-dlq",
"tenant-delete-dlq",
"tenant-delete-queue",
"tenant-updated-queue",
"tenant-updated-dlq",
"tenant-sync",
"tenant-sync-dlq",
"visual_layout_parsing_service_dead_letter_queue",
"visual_layout_parsing_service_queue",
"visual_layout_parsing_service_response_queue");
public QueueRenameMigration27(AmqpAdmin amqpAdmin) {
super(NAME, VERSION);
this.amqpAdmin = amqpAdmin;
}
@Override
protected void migrate() {
log.info("Migration: Deleting queues...");
for (String queueName : queueNames) {
log.info("Migration: Deleting queue {}", queueName);
amqpAdmin.deleteQueue(queueName);
}
}
}

70
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RuleFileUpdateMigration26.java Normal file
View File

@ -0,0 +1,70 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import java.util.List;
import java.util.Optional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class RuleFileUpdateMigration26 extends Migration {
private final DossierTemplateRepository dossierTemplateRepository;
private final RulesPersistenceService rulesPersistenceService;
private static final String NAME = "Migration for rule files, updating imports after package change";
private static final long VERSION = 26;
public RuleFileUpdateMigration26(DossierTemplateRepository dossierTemplateRepository, RulesPersistenceService rulesPersistenceService) {
super(NAME, VERSION);
this.dossierTemplateRepository = dossierTemplateRepository;
this.rulesPersistenceService = rulesPersistenceService;
}
@Override
protected void migrate() {
log.info("Migration: Updating all rule files in all dossier templates");
updateAllRuleFiles();
}
@SneakyThrows
private void updateAllRuleFiles() {
List<DossierTemplateEntity> dossierTemplates = dossierTemplateRepository.findAll();
List<RuleSetEntity> ruleSets = dossierTemplates.stream()
.map(dt -> rulesPersistenceService.getRules(dt.getId(), RuleFileType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.toList();
String tenantId = TenantContext.getTenantId();
ruleSets.parallelStream()
.forEach(ruleSet -> {
TenantContext.setTenantId(tenantId);
String updatedRules = ruleSet.getValue()
.replaceAll("import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;",
"import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;");
rulesPersistenceService.setRules(updatedRules, ruleSet.getDossierTemplateId(), RuleFileType.ENTITY);
});
}
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/PyInfraRequest.java
View File

@ -39,6 +39,11 @@ public class PyInfraRequest {
targetFilePath.put(key, StorageIdUtils.getStorageId(dossierUuid, fileUuid, fileType) + ".gz");
}
public void addTargetFilePath(String key, String fileName, String fileExtension) {
targetFilePath.put(key, StorageIdUtils.getStorageId(dossierUuid, fileUuid, fileName, fileExtension) + ".gz");
}
public void setResponseFilePath(FileType fileType) {

53
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.InputStream;
@ -24,6 +26,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.Ent
import com.iqser.red.storage.commons.exception.StorageException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
@ -194,6 +197,10 @@ public class FileManagementStorageService {
}
}
public boolean objectExists(String dossierId, String fileId, String fileName, String fileExtension) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileName, fileExtension));
}
public boolean objectExists(String dossierId, String fileId, FileType origin) {
@ -212,6 +219,44 @@ public class FileManagementStorageService {
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType));
}
public void deleteObject(String dossierId, String fileId, String fileName, String fileExtension) {
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileName, fileExtension));
}
public void deleteDocumentAndNerObjects(String dossierId, String fileId) {
deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE.name(), FileType.DOCUMENT_STRUCTURE_OLD.getExtension());
deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT.name(), FileType.DOCUMENT_TEXT_OLD.getExtension());
deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES.name(), FileType.DOCUMENT_PAGES_OLD.getExtension());
deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION.name(), FileType.DOCUMENT_POSITION_OLD.getExtension());
deleteObject(dossierId, fileId, FileType.DOCUMENT_CHUNKS);
deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
deleteObject(dossierId, fileId, FileType.LLM_NER_ENTITIES);
deleteObject(dossierId, fileId, FileType.AZURE_NER_ENTITIES);
}
public void deleteAllObjects(String dossierId, String fileId) {
deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
deleteEntityLog(dossierId, fileId);
deleteDocumentAndNerObjects(dossierId, fileId);
deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
deleteObject(dossierId, fileId, FileType.FIGURE);
deleteObject(dossierId, fileId, FileType.TABLES);
}
public void deleteEntityLog(String dossierId, String fileId) {
@ -229,4 +274,12 @@ public class FileManagementStorageService {
storageService.deleteObject(TenantContext.getTenantId(), storageId);
}
public DocumentStructureWrapper getDocumentStructure(String dossierId, String fileId) {
return new DocumentStructureWrapper(storageService.readProtoObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
DocumentStructure.parser()));
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java
View File

@ -9,9 +9,11 @@ import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Set;
import org.springframework.data.domain.PageRequest;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;

61
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -7,6 +7,8 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@ -311,7 +313,11 @@ public class FileStatusService {
return;
}
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_TEXT)) {
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_TEXT) && !fileManagementStorageService.objectExists(dossierId,
fileId,
FileType.DOCUMENT_TEXT.name(),
FileType.DOCUMENT_TEXT_OLD.getExtension())) {
var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, priority);
setStatusFullProcessing(fileId);
log.info("Add file: {} from dossier {} to layout parsing request queue", fileId, dossierId);
@ -328,7 +334,9 @@ public class FileStatusService {
}
if (settings.isLlmNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_CHUNKS)) {
var chunkingRequest = ChunkingRequestFactory.createChunkingRequest(dossierId, fileId);
var chunkingRequest = ChunkingRequestFactory.createChunkingRequest(dossierId,
fileId,
fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE));
setStatusFullProcessing(fileId);
log.info("Sending request to {} for {}/{}", CHUNKING_REQUEST_EXCHANGE, dossierId, fileId);
rabbitTemplate.convertAndSend(CHUNKING_REQUEST_EXCHANGE, TenantContext.getTenantId(), chunkingRequest);
@ -566,21 +574,36 @@ public class FileStatusService {
protected void addToLLMNerQueue(String dossierId, String fileId) {
setStatusNerAnalyzing(fileId);
boolean protoFilesExist = fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
BiFunction<FileType, FileType, String> getStorageId = (newFileType, oldFileType) -> protoFilesExist ? //
StorageIdUtils.getStorageId(dossierId, fileId, newFileType) : StorageIdUtils.getStorageId(dossierId, fileId, newFileType.name(), oldFileType.getExtension());
String chunksStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_CHUNKS);
String documentPagesStorageId = getStorageId.apply(FileType.DOCUMENT_PAGES, FileType.DOCUMENT_PAGES_OLD);
String documentStructureStorageId = getStorageId.apply(FileType.DOCUMENT_STRUCTURE, FileType.DOCUMENT_STRUCTURE_OLD);
String documentTextStorageId = getStorageId.apply(FileType.DOCUMENT_TEXT, FileType.DOCUMENT_TEXT_OLD);
String documentPositionStorageId = getStorageId.apply(FileType.DOCUMENT_POSITION, FileType.DOCUMENT_POSITION_OLD);
String resultStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES);
rabbitTemplate.convertAndSend(MessagingConfiguration.LLM_NER_REQUEST_EXCHANGE,
TenantContext.getTenantId(),
LlmNerMessage.builder()
.identifier(QueueMessageIdentifierService.buildIdentifier(dossierId, fileId, false))
.chunksStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_CHUNKS))
.documentPagesStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES))
.documentStructureStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE))
.documentTextStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT))
.documentPositionStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION))
.resultStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES))
.chunksStorageId(chunksStorageId)
.documentPagesStorageId(documentPagesStorageId)
.documentStructureStorageId(documentStructureStorageId)
.documentTextStorageId(documentTextStorageId)
.documentPositionStorageId(documentPositionStorageId)
.resultStorageId(resultStorageId)
.build(),
message -> {
message.getMessageProperties().setPriority(1);
return message;
});
}
@ -880,14 +903,7 @@ public class FileStatusService {
if (requiresStructureAnalysis) {
log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_CHUNKS);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.LLM_NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.AZURE_NER_ENTITIES);
fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId);
}
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), AnalysisType.DEFAULT);
@ -923,17 +939,8 @@ public class FileStatusService {
public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean keepManualRedactions, boolean disableAutomaticAnalysis) {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ORIGIN);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.AZURE_NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.FIGURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TABLES);
fileManagementStorageService.deleteAllObjects(dossierId, fileId);
if (keepManualRedactions) {
fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, true, disableAutomaticAnalysis);

19
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java
View File

@ -1,7 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -14,9 +12,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ImageMongoService;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@ -33,15 +30,14 @@ public class ImageSimilarityService {
ObjectMapper objectMapper;
public void saveImages(String templateId, String dossierId, String fileId, String storageId) throws IOException {
public void saveImages(String templateId, String dossierId, String fileId) {
DocumentStructureWrapper documentStructure = fileManagementStorageService.getDocumentStructure(dossierId, fileId);
List<ImageDocument> imageDocuments = new ArrayList<>();
try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) {
DocumentStructure documentStructure = objectMapper.readValue(inputStream, DocumentStructure.class);
documentStructure.streamAllEntries()
.filter(entry -> entry.getType().equals(NodeType.IMAGE))
.forEach(i -> {
Map<String, String> properties = i.getProperties();
Map<String, String> properties = i.getPropertiesMap();
ImageDocument imageDocument = new ImageDocument();
imageDocument.setImageId(properties.get("id"));
imageDocument.setFeatureVector(parseRepresentationVector(properties.get("representationHash")));
@ -51,7 +47,7 @@ public class ImageSimilarityService {
imageDocuments.add(imageDocument);
});
}
if (imageDocuments.isEmpty()) {
return;
}
@ -64,8 +60,7 @@ public class ImageSimilarityService {
ImageDocument centralImage = this.imageMongoService.findById(annotationId);
List<ImageDocument> similarImages = this.findSimilarImages(centralImage, distance, scope);
log.info("received similar images: {}", similarImages);
return similarImages.stream()
.collect(Collectors.toList());
return new ArrayList<>(similarImages);
}

283
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DocumentDataFilesConversionJob.java Normal file
View File

@ -0,0 +1,283 @@
package com.iqser.red.service.persistence.management.v1.processor.service.job;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.primitives.Floats;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Service
@DisallowConcurrentExecution
public class DocumentDataFilesConversionJob implements Job {
private final FileRepository fileRepository;
private final FileStatusPersistenceService fileStatusPersistenceService;
private final TenantProvider tenantProvider;
private final FileManagementStorageService fileManagementStorageService;
private final StorageService storageService;
private final ObjectMapper objectMapper;
private static final long MAX_JOB_DURATION = 2 * 60 * 60 * 1000L;
private static final int FILE_BATCH_SIZE = 32;
@Override
public void execute(JobExecutionContext jobExecutionContext) {
long startTime = System.currentTimeMillis();
tenantProvider.getTenants()
.forEach(tenant -> {
if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
return;
}
TenantContext.setTenantId(tenant.getTenantId());
convertAllDocumentDataFiles(startTime);
});
}
@SneakyThrows
private void convertAllDocumentDataFiles(long startTime) {
while (System.currentTimeMillis() - startTime < MAX_JOB_DURATION) {
List<FileEntity> files = fileRepository.findByProtobufMigrationDoneFalse(PageRequest.of(0, FILE_BATCH_SIZE));
if (files.isEmpty()) {
break;
}
String tenantId = TenantContext.getTenantId();
files.parallelStream()
.forEach(file -> {
TenantContext.setTenantId(tenantId);
log.info("Reading dossier {} file {} from storage", file.getDossierId(), file.getId());
String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.DOCUMENT_STRUCTURE.name(), ".json");
String newDocumentStructureStorageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.DOCUMENT_STRUCTURE);
try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), oldDocumentStructureStorageId)) {
DocumentStructure oldDocumentStructure = objectMapper.readValue(inputStream, DocumentStructure.class);
DocumentStructureProto.DocumentStructure newDocumentStructure = convertDocumentStructure(oldDocumentStructure);
storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentStructureStorageId, newDocumentStructure);
//storageService.deleteObject(TenantContext.getTenantId(), oldDocumentStructureStorageId);
} catch (StorageObjectDoesNotExist e) {
log.info("No document structure found for {}, {}, ignoring....", file.getDossierId(), file.getId());
} catch (IOException e) {
log.error("Document structure could not be mapped for {}, {}, ignoring....", file.getDossierId(), file.getId());
}
String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.DOCUMENT_PAGES.name(), ".json");
String newDocumentPagesStorageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.DOCUMENT_PAGES);
try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), oldDocumentPagesStorageId)) {
DocumentPage[] oldDocumentPages = objectMapper.readValue(inputStream, DocumentPage[].class);
DocumentPageProto.AllDocumentPages newDocumentPages = convertAllDocumentPages(oldDocumentPages);
storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentPagesStorageId, newDocumentPages);
//storageService.deleteObject(TenantContext.getTenantId(), oldDocumentPagesStorageId);
} catch (StorageObjectDoesNotExist e) {
log.info("No document pages found for {}, {}, ignoring....", file.getDossierId(), file.getId());
} catch (IOException e) {
log.error("Document pages could not be mapped for {}, {}, ignoring....", file.getDossierId(), file.getId());
}
String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.DOCUMENT_POSITION.name(), ".json");
String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.DOCUMENT_POSITION);
try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId)) {
DocumentPositionData[] oldDocumentPositionData = objectMapper.readValue(inputStream, DocumentPositionData[].class);
DocumentPositionDataProto.AllDocumentPositionData newDocumentPositionData = convertAllDocumentPositionData(oldDocumentPositionData);
storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentPositionDataStorageId, newDocumentPositionData);
//storageService.deleteObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId);
} catch (StorageObjectDoesNotExist e) {
log.info("No document position data found for {}, {}, ignoring....", file.getDossierId(), file.getId());
} catch (IOException e) {
log.error("Document position data could not be mapped for {}, {}, ignoring....", file.getDossierId(), file.getId());
}
String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.DOCUMENT_TEXT.name(), ".json");
String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.DOCUMENT_TEXT);
try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId)) {
DocumentTextData[] oldDocumentTextData = objectMapper.readValue(inputStream, DocumentTextData[].class);
DocumentTextDataProto.AllDocumentTextData newDocumentTextData = convertAllDocumentTextData(oldDocumentTextData);
storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentTextDataStorageId, newDocumentTextData);
//storageService.deleteObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId);
} catch (StorageObjectDoesNotExist e) {
log.info("No document text data found for {}, {}, ignoring....", file.getDossierId(), file.getId());
} catch (IOException e) {
log.error("Document text data could not be mapped for {}, {}, ignoring....", file.getDossierId(), file.getId());
}
});
fileStatusPersistenceService.setProtobufMigrationDoneForFiles(files.stream()
.map(FileEntity::getId)
.toList());
}
}
private static EntryDataProto.EntryData convertEntryData(DocumentStructure.EntryData oldEntryData) {
EntryDataProto.EntryData.Builder builder = EntryDataProto.EntryData.newBuilder();
builder.setType(NodeTypeProto.NodeType.valueOf(oldEntryData.getType().name()));
builder.addAllTreeId(Arrays.stream(oldEntryData.getTreeId()).boxed()
.collect(Collectors.toList()));
builder.addAllAtomicBlockIds(Arrays.asList(oldEntryData.getAtomicBlockIds()));
builder.addAllPageNumbers(Arrays.asList(oldEntryData.getPageNumbers()));
builder.putAllProperties(oldEntryData.getProperties());
if (oldEntryData.getChildren() != null) {
oldEntryData.getChildren()
.forEach(child -> builder.addChildren(convertEntryData(child)));
}
builder.addAllEngines(oldEntryData.getEngines()
.stream()
.map(DocumentDataFilesConversionJob::convertLayoutEngine)
.toList());
return builder.build();
}
private static LayoutEngineProto.LayoutEngine convertLayoutEngine(LayoutEngine oldLayoutEngine) {
return LayoutEngineProto.LayoutEngine.valueOf(oldLayoutEngine.name());
}
private static DocumentStructureProto.DocumentStructure convertDocumentStructure(DocumentStructure oldStructure) {
DocumentStructureProto.DocumentStructure.Builder newBuilder = DocumentStructureProto.DocumentStructure.newBuilder();
if (oldStructure.getRoot() != null) {
newBuilder.setRoot(convertEntryData(oldStructure.getRoot()));
}
return newBuilder.build();
}
private static DocumentPageProto.DocumentPage convertDocumentPage(DocumentPage oldPage) {
return DocumentPageProto.DocumentPage.newBuilder()
.setNumber(oldPage.getNumber())
.setHeight(oldPage.getHeight())
.setWidth(oldPage.getWidth())
.setRotation(oldPage.getRotation())
.build();
}
private static DocumentPageProto.AllDocumentPages convertAllDocumentPages(DocumentPage[] oldPages) {
DocumentPageProto.AllDocumentPages.Builder allPagesBuilder = DocumentPageProto.AllDocumentPages.newBuilder();
for (DocumentPage oldPage : oldPages) {
DocumentPageProto.DocumentPage newPage = convertDocumentPage(oldPage);
allPagesBuilder.addDocumentPages(newPage);
}
return allPagesBuilder.build();
}
private static DocumentPositionDataProto.DocumentPositionData convertDocumentPositionData(DocumentPositionData oldData) {
DocumentPositionDataProto.DocumentPositionData.Builder builder = DocumentPositionDataProto.DocumentPositionData.newBuilder()
.setId(oldData.getId())
.addAllStringIdxToPositionIdx(Arrays.stream(oldData.getStringIdxToPositionIdx()).boxed()
.collect(Collectors.toList()));
for (float[] pos : oldData.getPositions()) {
DocumentPositionDataProto.DocumentPositionData.Position position = DocumentPositionDataProto.DocumentPositionData.Position.newBuilder()
.addAllValue(Floats.asList(pos))
.build();
builder.addPositions(position);
}
return builder.build();
}
private static DocumentPositionDataProto.AllDocumentPositionData convertAllDocumentPositionData(DocumentPositionData[] oldDataList) {
DocumentPositionDataProto.AllDocumentPositionData.Builder allDataBuilder = DocumentPositionDataProto.AllDocumentPositionData.newBuilder();
for (DocumentPositionData oldData : oldDataList) {
allDataBuilder.addDocumentPositionData(convertDocumentPositionData(oldData));
}
return allDataBuilder.build();
}
private static DocumentTextDataProto.DocumentTextData convertDocumentTextData(DocumentTextData oldData) {
DocumentTextDataProto.DocumentTextData.Builder builder = DocumentTextDataProto.DocumentTextData.newBuilder()
.setId(oldData.getId())
.setPage(oldData.getPage())
.setSearchText(oldData.getSearchText())
.setNumberOnPage(oldData.getNumberOnPage())
.setStart(oldData.getStart())
.setEnd(oldData.getEnd())
.addAllLineBreaks(Arrays.stream(oldData.getLineBreaks()).boxed()
.collect(Collectors.toList()));
return builder.build();
}
private static DocumentTextDataProto.AllDocumentTextData convertAllDocumentTextData(DocumentTextData[] oldDataList) {
DocumentTextDataProto.AllDocumentTextData.Builder allDataBuilder = DocumentTextDataProto.AllDocumentTextData.newBuilder();
for (DocumentTextData oldData : oldDataList) {
allDataBuilder.addDocumentTextData(convertDocumentTextData(oldData));
}
return allDataBuilder.build();
}
}

19
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/ChunkingRequestFactory.java
View File

@ -14,13 +14,22 @@ public class ChunkingRequestFactory {
private static final String STRUCTURE = "structure";
public PyInfraRequest createChunkingRequest(String dossierId, String fileId) {
public PyInfraRequest createChunkingRequest(String dossierId, String fileId, boolean protoFileExists) {
PyInfraRequest pyInfraRequest = new PyInfraRequest(dossierId, fileId);
pyInfraRequest.addTargetFilePath(PAGES, FileType.DOCUMENT_PAGES);
pyInfraRequest.addTargetFilePath(TEXT_BLOCKS, FileType.DOCUMENT_TEXT);
pyInfraRequest.addTargetFilePath(TEXT_POSITIONS, FileType.DOCUMENT_POSITION);
pyInfraRequest.addTargetFilePath(STRUCTURE, FileType.DOCUMENT_STRUCTURE);
if (protoFileExists) {
pyInfraRequest.addTargetFilePath(PAGES, FileType.DOCUMENT_PAGES);
pyInfraRequest.addTargetFilePath(TEXT_BLOCKS, FileType.DOCUMENT_TEXT);
pyInfraRequest.addTargetFilePath(TEXT_POSITIONS, FileType.DOCUMENT_POSITION);
pyInfraRequest.addTargetFilePath(STRUCTURE, FileType.DOCUMENT_STRUCTURE);
} else {
pyInfraRequest.addTargetFilePath(PAGES, FileType.DOCUMENT_PAGES.name(), FileType.DOCUMENT_PAGES_OLD.getExtension());
pyInfraRequest.addTargetFilePath(TEXT_BLOCKS, FileType.DOCUMENT_TEXT.name(), FileType.DOCUMENT_TEXT_OLD.getExtension());
pyInfraRequest.addTargetFilePath(TEXT_POSITIONS, FileType.DOCUMENT_POSITION.name(), FileType.DOCUMENT_POSITION_OLD.getExtension());
pyInfraRequest.addTargetFilePath(STRUCTURE, FileType.DOCUMENT_STRUCTURE.name(), FileType.DOCUMENT_STRUCTURE_OLD.getExtension());
}
pyInfraRequest.setResponseFilePath(FileType.DOCUMENT_CHUNKS);
return pyInfraRequest;
}

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -690,5 +690,12 @@ public class FileStatusPersistenceService {
}
@Transactional
public void setProtobufMigrationDoneForFiles(List<String> fileIds) {
fileRepository.setProtobufMigrationDone(fileIds);
}
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -4,6 +4,7 @@ import java.time.OffsetDateTime;
import java.util.List;
import java.util.Optional;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
@ -421,6 +422,16 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("SELECT f FROM FileEntity f WHERE f.hardDeletedTime IS NULL AND f.deleted IS NOT NULL AND f.deleted < :deletionThreshold")
List<FileEntity> findFilesSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold);
@Query("SELECT f FROM FileEntity f WHERE f.protobufMigrationDone = false")
List<FileEntity> findByProtobufMigrationDoneFalse(Pageable pageable);
@Modifying
@Query("update FileEntity f set f.protobufMigrationDone = true "
+ "where f.id in (:fileIds)")
void setProtobufMigrationDone(@Param("fileIds") List<String> fileIds);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java
View File

@ -70,7 +70,7 @@ public class LayoutParsingFinishedMessageReceiver {
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.LAYOUT_UPDATE, fileStatusService.getStatus(fileId).getNumberOfAnalyses() + 1);
try {
imageSimilarityService.saveImages(templateId, dossierId, fileId, storageId);
imageSimilarityService.saveImages(templateId, dossierId, fileId);
} catch (Exception e) {
log.error("Error occured during save images: {} ", e.getMessage(), e);
throw e;

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/StorageIdUtils.java
View File

@ -12,6 +12,11 @@ public final class StorageIdUtils {
return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
}
public static String getStorageId(String dossierId, String fileId, String fileName, String fileExtension) {
return dossierId + "/" + fileId + "." + fileName + fileExtension;
}
public static String getStorageId(String userId, String dossierId, String filename) {

4
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -228,4 +228,6 @@ databaseChangeLog:
- include:
file: db/changelog/tenant/142-add-date-formats-table.yaml
- include:
file: db/changelog/tenant/143-modify-download-redaction-file-status-details.yaml
file: db/changelog/tenant/143-modify-download-redaction-file-status-details.yaml
- include:
file: db/changelog/tenant/144-add-protobuf-migration-done-to-file.yaml

12
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/144-add-protobuf-migration-done-to-file.yaml Normal file
View File

@ -0,0 +1,12 @@
databaseChangeLog:
- changeSet:
id: add-protobuf-migration-done-to-file
author: maverick
changes:
- addColumn:
columns:
- column:
name: protobuf_migration_done
type: BOOLEAN
defaultValueBoolean: false
tableName: file

3
persistence-service-v1/persistence-service-server-v1/build.gradle.kts
View File

@ -23,7 +23,7 @@ dependencies {
api(project(":persistence-service-external-api-impl-v2"))
api(project(":persistence-service-internal-api-impl-v1"))
api(project(":persistence-service-shared-mongo-v1"))
api("com.iqser.red.commons:storage-commons:2.49.0")
api("com.iqser.red.commons:storage-commons:2.50.0")
api("junit:junit:4.13.2")
api("org.apache.logging.log4j:log4j-slf4j-impl:2.20.0")
api("net.logstash.logback:logstash-logback-encoder:7.4")
@ -34,6 +34,7 @@ dependencies {
testImplementation("org.testcontainers:postgresql:1.17.1")
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
testImplementation("com.yannbriancon:spring-hibernate-query-utils:2.0.0")
testImplementation("com.google.protobuf:protobuf-java:4.27.1")
}
description = "persistence-service-server-v1"

1
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
View File

@ -72,7 +72,6 @@ import lombok.extern.slf4j.Slf4j;
@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, CassandraAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
@Import({PersistenceServiceExternalApiConfigurationV2.class, PersistenceServiceExternalApiConfiguration.class, PersistenceServiceInternalApiConfiguration.class, PersistenceServiceExternalApiCacheConfiguration.class, MultiTenancyWebConfiguration.class, PersistenceServiceProcessorConfiguration.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class})
@EnableAspectJAutoProxy
public class Application implements ApplicationContextAware {
/**

223
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@ -11,6 +12,8 @@ import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
@ -24,6 +27,8 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.InputStreamResource;
import org.springframework.mock.web.MockMultipartFile;
import com.github.dockerjava.zerodep.shaded.org.apache.commons.codec.binary.Base64;
@ -51,8 +56,10 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileDel
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DeletedFilesCleanupJob;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DocumentDataFilesConversionJob;
import com.iqser.red.service.persistence.management.v1.processor.service.job.SoftDeletedFilesCleanupJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
@ -80,6 +87,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Forc
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import feign.FeignException;
@ -147,6 +159,9 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
@Autowired
private SoftDeletedFilesCleanupJob softDeletedFilesCleanupJob;
@Autowired
private DocumentDataFilesConversionJob documentDataFilesConversionJob;
@Test
public void testFileSoftDeleteReupload() {
@ -894,7 +909,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertNotNull(fileStatusService.getDossierStatus(dossierId)
.get(0).getDeleted());
assertNotNull(fileStatusService.getDossierStatus(dossierId)
.get(1).getHardDeletedTime());
.get(1).getHardDeletedTime());
assertNotNull(fileStatusService.getDossierStatus(dossierId)
.get(1).getDeleted());
@ -909,7 +924,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertNotNull(otherFileModel.get().getHardDeletedTime());
assertNotNull(otherFileModel.get().getDeleted());
Optional<FileModel> fileModel = fileStatusService.getDossierStatus(dossierId)
.stream()
.filter(fm -> fm.getId().equals(fileId))
@ -977,4 +991,209 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
deletedFilesCleanupJob.execute(null);
}
@Test
@SneakyThrows
public void testDocumentDataFilesConversionJobSuccessful() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
String dossierId = dossier.getId();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
String fileId = file.getId();
String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE.name(), ".json");
String newDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES.name(), ".json");
String newDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES);
String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION.name(), ".json");
String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION);
String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT.name(), ".json");
String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT);
ClassPathResource oldDocumentStructureClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_STRUCTURE.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentStructureStorageId, oldDocumentStructureClassPath.getInputStream());
ClassPathResource oldDocumentPagesClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_PAGES.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentPagesStorageId, oldDocumentPagesClassPath.getInputStream());
ClassPathResource oldDocumentPositionDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_POSITION.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId, oldDocumentPositionDataClassPath.getInputStream());
ClassPathResource oldDocumentTextDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_TEXT.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId, oldDocumentTextDataClassPath.getInputStream());
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
documentDataFilesConversionJob.execute(null);
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE));
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_TEXT));
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_PAGES));
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_POSITION));
DocumentStructureProto.DocumentStructure documentStructure = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentStructureStorageId,
DocumentStructureProto.DocumentStructure.parser());
assertEquals(documentStructure.getRoot().getPageNumbersList().size(), 9);
DocumentPageProto.AllDocumentPages allDocumentPages = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentPagesStorageId,
DocumentPageProto.AllDocumentPages.parser());
assertEquals(allDocumentPages.getDocumentPagesList().size(), 9);
DocumentTextDataProto.AllDocumentTextData allDocumentTextData = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentTextDataStorageId,
DocumentTextDataProto.AllDocumentTextData.parser());
assertEquals(allDocumentTextData.getDocumentTextDataCount(), 114);
DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentPositionDataStorageId,
DocumentPositionDataProto.AllDocumentPositionData.parser());
assertEquals(allDocumentPositionData.getDocumentPositionDataList().size(), 114);
}
@Test
@SneakyThrows
@Disabled
public void serializeDocumentDataFiles() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
String dossierId = dossier.getId();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
String fileId = file.getId();
String newDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
String newDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES);
String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION);
String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT);
ClassPathResource newDocumentStructureClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_STRUCTURE.proto");
storageService.storeObject(TenantContext.getTenantId(), newDocumentStructureStorageId, newDocumentStructureClassPath.getInputStream());
ClassPathResource newDocumentPagesClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_PAGES.proto");
storageService.storeObject(TenantContext.getTenantId(), newDocumentPagesStorageId, newDocumentPagesClassPath.getInputStream());
ClassPathResource newDocumentPositionDataClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_POSITION.proto");
storageService.storeObject(TenantContext.getTenantId(), newDocumentPositionDataStorageId, newDocumentPositionDataClassPath.getInputStream());
ClassPathResource newDocumentTextDataClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_TEXT.proto");
storageService.storeObject(TenantContext.getTenantId(), newDocumentTextDataStorageId, newDocumentTextDataClassPath.getInputStream());
DocumentStructureProto.DocumentStructure documentStructure = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentStructureStorageId,
DocumentStructureProto.DocumentStructure.parser());
DocumentPageProto.AllDocumentPages allDocumentPages = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentPagesStorageId,
DocumentPageProto.AllDocumentPages.parser());
DocumentTextDataProto.AllDocumentTextData allDocumentTextData = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentTextDataStorageId,
DocumentTextDataProto.AllDocumentTextData.parser());
DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentPositionDataStorageId,
DocumentPositionDataProto.AllDocumentPositionData.parser());
}
@Test
@SneakyThrows
@Disabled
public void testDocumentDataFilesConversionAndSaveToTempDir() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
String dossierId = dossier.getId();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
String fileId = file.getId();
String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE.name(), ".json");
String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES.name(), ".json");
String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION.name(), ".json");
String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT.name(), ".json");
String newDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
String newDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES);
String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION);
String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT);
ClassPathResource oldDocumentStructureClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_STRUCTURE.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentStructureStorageId, oldDocumentStructureClassPath.getInputStream());
ClassPathResource oldDocumentPagesClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_PAGES.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentPagesStorageId, oldDocumentPagesClassPath.getInputStream());
ClassPathResource oldDocumentPositionDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_POSITION.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId, oldDocumentPositionDataClassPath.getInputStream());
ClassPathResource oldDocumentTextDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_TEXT.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId, oldDocumentTextDataClassPath.getInputStream());
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
documentDataFilesConversionJob.execute(null);
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE));
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_TEXT));
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_PAGES));
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_POSITION));
DocumentStructureProto.DocumentStructure documentStructure = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentStructureStorageId,
DocumentStructureProto.DocumentStructure.parser());
DocumentPageProto.AllDocumentPages allDocumentPages = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentPagesStorageId,
DocumentPageProto.AllDocumentPages.parser());
DocumentTextDataProto.AllDocumentTextData allDocumentTextData = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentTextDataStorageId,
DocumentTextDataProto.AllDocumentTextData.parser());
DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentPositionDataStorageId,
DocumentPositionDataProto.AllDocumentPositionData.parser());
String tempDir = "/tmp";
Path documentStructurePath = Path.of(tempDir, "new.DOCUMENT_STRUCTURE.proto");
Files.write(documentStructurePath, documentStructure.toByteArray());
Path documentPagesPath = Path.of(tempDir, "new.DOCUMENT_PAGES.proto");
Files.write(documentPagesPath, allDocumentPages.toByteArray());
Path documentTextPath = Path.of(tempDir, "new.DOCUMENT_TEXT.proto");
Files.write(documentTextPath, allDocumentTextData.toByteArray());
Path documentPositionPath = Path.of(tempDir, "new.DOCUMENT_POSITION.proto");
Files.write(documentPositionPath, allDocumentPositionData.toByteArray());
Assertions.assertTrue(Files.exists(documentStructurePath));
Assertions.assertTrue(Files.exists(documentPagesPath));
Assertions.assertTrue(Files.exists(documentTextPath));
Assertions.assertTrue(Files.exists(documentPositionPath));
}
@Test
@Disabled
public void testParseProtoFiles() throws Exception {
String tempDir = "/tmp";
Path documentStructurePath = Path.of(tempDir, "new.DOCUMENT_STRUCTURE.proto");
Path documentPagesPath = Path.of(tempDir, "new.DOCUMENT_PAGES.proto");
Path documentTextPath = Path.of(tempDir, "new.DOCUMENT_TEXT.proto");
Path documentPositionPath = Path.of(tempDir, "new.DOCUMENT_POSITION.proto");
byte[] documentStructureBytes = Files.readAllBytes(documentStructurePath);
byte[] documentPagesBytes = Files.readAllBytes(documentPagesPath);
byte[] documentTextBytes = Files.readAllBytes(documentTextPath);
byte[] documentPositionBytes = Files.readAllBytes(documentPositionPath);
DocumentStructureProto.DocumentStructure documentStructure =
DocumentStructureProto.DocumentStructure.parseFrom(documentStructureBytes);
DocumentPageProto.AllDocumentPages allDocumentPages =
DocumentPageProto.AllDocumentPages.parseFrom(documentPagesBytes);
DocumentTextDataProto.AllDocumentTextData allDocumentTextData =
DocumentTextDataProto.AllDocumentTextData.parseFrom(documentTextBytes);
DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData =
DocumentPositionDataProto.AllDocumentPositionData.parseFrom(documentPositionBytes);
}
}

BIN
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/fileexchange/EFSA sanitisation GFL v1_file-exchange.zip
View File

Binary file not shown.

56
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/oldDocumentData/old.DOCUMENT_PAGES.json Normal file
View File

@ -0,0 +1,56 @@
[
{
"number": 1,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 2,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 3,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 4,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 5,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 6,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 7,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 8,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 9,
"height": 792,
"width": 612,
"rotation": 0
}
]

59458
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/oldDocumentData/old.DOCUMENT_POSITION.json Normal file
View File

File diff suppressed because it is too large Load Diff

2946
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/oldDocumentData/old.DOCUMENT_STRUCTURE.json Normal file
View File

File diff suppressed because it is too large Load Diff

1225
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/oldDocumentData/old.DOCUMENT_TEXT.json Normal file
View File

File diff suppressed because it is too large Load Diff

12
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java
View File

@ -23,10 +23,14 @@ public enum FileType {
VISUAL_LAYOUT(".json"),
COMPONENTS(".json"),
// document is split into 4 files, all should be overridden/deleted at the same time
DOCUMENT_TEXT(".json"),
DOCUMENT_STRUCTURE(".json"),
DOCUMENT_POSITION(".json"),
DOCUMENT_PAGES(".json"),
DOCUMENT_TEXT_OLD(".json"),
DOCUMENT_STRUCTURE_OLD(".json"),
DOCUMENT_POSITION_OLD(".json"),
DOCUMENT_PAGES_OLD(".json"),
DOCUMENT_TEXT(".proto"),
DOCUMENT_STRUCTURE(".proto"),
DOCUMENT_POSITION(".proto"),
DOCUMENT_PAGES(".proto"),
DOCUMENT_CHUNKS(".json"),
ENTITY_LOG(".json"),
ENTITY_LOG_BAK(".json"),

4
persistence-service-v1/persistence-service-shared-mongo-v1/build.gradle.kts
View File

@ -9,7 +9,9 @@ dependencies {
api(project(":persistence-service-shared-api-v1"))
api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.2")
api("com.google.guava:guava:31.1-jre")
api("com.knecon.fforesight:mongo-database-commons:0.13.0+RED9348.0")
api("com.knecon.fforesight:mongo-database-commons:0.13.0+RED9348.0") {
exclude(group = "com.iqser.red.commons", module = "storage-commons")
}
api("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-validation:3.1.3")
testImplementation("com.iqser.red.commons:test-commons:2.1.0")

43
persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogDocumentUpdateService.java
View File

@ -1,5 +1,8 @@
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.service;
import java.util.ArrayList;
import java.util.List;
import org.bson.Document;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
@ -9,11 +12,14 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.mapper.EntityLogDocumentMapper;
import com.mongodb.DBRef;
@Service
public class EntityLogDocumentUpdateService {
public static final String ENTITY_LOG_ENTRY_DOCUMENTS = "entityLogEntryDocuments";
private final MongoTemplate mongoTemplate;
private final EntityLogDocumentMapper mapper = EntityLogDocumentMapper.INSTANCE;
@ -25,10 +31,43 @@ public class EntityLogDocumentUpdateService {
Document doc = new Document();
mongoTemplate.getConverter().write(mapper.toLogDocument(dossierId, fileId, entityLog), doc);
doc.remove("entityLogEntryDocuments");
doc.remove(ENTITY_LOG_ENTRY_DOCUMENTS);
Update update = new Update();
doc.forEach(update::set);
mongoTemplate.updateFirst(Query.query(Criteria.where("_id").is(mapper.getLogId(dossierId, fileId))), update, EntityLogDocument.class);
mongoTemplate.updateFirst(getIdQuery(mapper.getLogId(dossierId, fileId)), update, EntityLogDocument.class);
}
public void addEntityLogEntryReferences(String entityLogId, List<EntityLogEntryDocument> newEntries) {
List<DBRef> dbRefs = convertEntriesToDbRefs(newEntries);
Update update = new Update().push(ENTITY_LOG_ENTRY_DOCUMENTS).each(dbRefs.toArray());
mongoTemplate.updateFirst(getIdQuery(entityLogId), update, EntityLogDocument.class);
}
public void removeEntityLogEntryReferences(String entityLogId, List<EntityLogEntryDocument> entriesToRemove) {
List<DBRef> dbRefs = convertEntriesToDbRefs(entriesToRemove);
Update update = new Update().pullAll(ENTITY_LOG_ENTRY_DOCUMENTS, dbRefs.toArray());
mongoTemplate.updateFirst(getIdQuery(entityLogId), update, EntityLogDocument.class);
}
private static List<DBRef> convertEntriesToDbRefs(List<EntityLogEntryDocument> entriesToRemove) {
List<DBRef> dbRefs = new ArrayList<>();
for (EntityLogEntryDocument entry : entriesToRemove) {
DBRef dbRef = new DBRef("entity-log-entries", entry.getId());
dbRefs.add(dbRef);
}
return dbRefs;
}
private Query getIdQuery(String mapper) {
return Query.query(Criteria.where("_id").is(mapper));
}
}

57
persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java
View File

@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.mapper.Enti
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentCustomRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
@ -49,10 +50,13 @@ public class EntityLogMongoService {
EntityLogDocument entityLogDocument = entityLogDocumentRepository.save(mapper.toLogDocument(dossierId, fileId, entityLog));
entityLogEntryDocumentRepository.saveAll(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
.toList());
String tenantId = TenantContext.getTenantId();
entityLog.getEntityLogEntry().parallelStream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
.forEach(entity -> {
TenantContext.setTenantId(tenantId);
entityLogEntryDocumentRepository.save(entity);
});
}
@ -102,16 +106,13 @@ public class EntityLogMongoService {
String entityLogId = mapper.getLogId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocuments().addAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.insert(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
entityLogDocumentUpdateService.addEntityLogEntryReferences(entityLogId, entityLogEntryDocuments);
}
@ -119,16 +120,16 @@ public class EntityLogMongoService {
String entityLogId = mapper.getLogId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
String tenantId = TenantContext.getTenantId();
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.parallelStream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogId, entityLogEntry))
.peek(entity -> {
TenantContext.setTenantId(tenantId);
entityLogEntryDocumentRepository.save(entity);
})
.toList();
entityLogDocument.getEntityLogEntryDocuments().addAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.saveAll(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
entityLogDocumentUpdateService.addEntityLogEntryReferences(entityLogId, entityLogEntryDocuments);
}
@ -136,9 +137,13 @@ public class EntityLogMongoService {
String entityLogId = mapper.getLogId(dossierId, fileId);
entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogId, entityLogEntry))
.toList());
String tenantId = TenantContext.getTenantId();
entityLogEntries.parallelStream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogId, entityLogEntry))
.forEach(entity -> {
TenantContext.setTenantId(tenantId);
entityLogEntryDocumentRepository.save(entity);
});
}
@ -146,16 +151,16 @@ public class EntityLogMongoService {
String entityLogId = mapper.getLogId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
String tenantId = TenantContext.getTenantId();
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.parallelStream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogId, entityLogEntry))
.peek(entity -> {
TenantContext.setTenantId(tenantId);
entityLogEntryDocumentRepository.delete(entity);
})
.toList();
entityLogDocument.getEntityLogEntryDocuments().removeAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
entityLogDocumentUpdateService.removeEntityLogEntryReferences(entityLogId, entityLogEntryDocuments);
}