RED-9255: fix annotations, refactor for observability #562
@ -0,0 +1,70 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class DossierExportService {
|
||||
|
||||
DictionaryPersistenceService dictionaryPersistenceService;
|
||||
FileExportService fileExportService;
|
||||
FileStatusManagementService fileStatusManagementService;
|
||||
EntityTypeExportService entityTypeExportService;
|
||||
ObjectMapper mapper;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
|
||||
public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
|
||||
|
||||
List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
|
||||
|
||||
if (!request.dossierIds().contains(dossier.getId()) //
|
||||
&& files.stream()
|
||||
.noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
|
||||
// dossier has no files in requested files and dossier not explicitly requested -> don't export it.
|
||||
return;
|
||||
}
|
||||
|
||||
Path dossierFolder = folder.resolve(dossier.getId());
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
|
||||
|
||||
for (FileModel fileEntity : files) {
|
||||
if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
|
||||
continue;
|
||||
}
|
||||
fileExportService.addFileToArchive(archiver, dossierFolder, request, fileEntity);
|
||||
}
|
||||
|
||||
List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
|
||||
|
||||
String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
|
||||
|
||||
for (TypeEntity type : types) {
|
||||
entityTypeExportService.addEntityTypeToArchive(archiver, type, entitiesFolder);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,86 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
|
||||
|
||||
import io.micrometer.observation.ObservationRegistry;
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class DossierImportService {
|
||||
|
||||
DossierCreatorService dossierCreatorService;
|
||||
FileImportService fileImportService;
|
||||
EntityTypeImportService entityTypeImportService;
|
||||
ObservationRegistry registry;
|
||||
|
||||
|
||||
@Observed(name = "DossierImportService", contextualName = "import-dossier")
|
||||
public void importDossier(FileExchangeImportModel fileExchangeImportModel,
|
||||
String userId,
|
||||
Dossier dossierToImport,
|
||||
TemplateImportInfo templateImportInfo,
|
||||
List<String> allReportTemplateIds) {
|
||||
|
||||
String dossierId = getDossierForImport(templateImportInfo, dossierToImport, userId, allReportTemplateIds);
|
||||
|
||||
entityTypeImportService.importEntityTypes(templateImportInfo.getDossierTemplateId(),
|
||||
dossierId,
|
||||
fileExchangeImportModel.getDossierTypes()
|
||||
.get(dossierToImport.getId()));
|
||||
|
||||
List<FileExchangeImportModel.FileImport> files = fileExchangeImportModel.getFiles().getOrDefault(dossierToImport.getId(), Collections.emptyList());
|
||||
|
||||
fileImportService.importFilesInParallel(files, userId, templateImportInfo, dossierId);
|
||||
}
|
||||
|
||||
|
||||
private String getDossierForImport(TemplateImportInfo templateImportInfo, Dossier dossier, String userId, List<String> reportTemplateIds) {
|
||||
|
||||
String dossierTemplateId = templateImportInfo.getDossierTemplateId();
|
||||
|
||||
CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
|
||||
.dossierName(dossier.getName())
|
||||
.description(dossier.getDescription())
|
||||
.downloadFileTypes(dossier.getDownloadFileTypes())
|
||||
.dueDate(dossier.getDueDate())
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.requestingUser(userId)
|
||||
.reportTemplateIds(reportTemplateIds)
|
||||
.build();
|
||||
|
||||
Dossier importedDossier = null;
|
||||
int retries = 0;
|
||||
while (importedDossier == null && retries < 10) {
|
||||
try {
|
||||
importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
|
||||
} catch (ConflictException e) {
|
||||
retries++;
|
||||
request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
|
||||
}
|
||||
}
|
||||
if (importedDossier == null) {
|
||||
throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
|
||||
}
|
||||
return importedDossier.getId();
|
||||
}
|
||||
|
||||
}
|
||||
@ -67,6 +67,7 @@ import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
@ -120,7 +121,7 @@ public class DossierTemplateImportService {
|
||||
return dossierTemplateArchiveReader.buildResult();
|
||||
}
|
||||
|
||||
|
||||
@Observed(name = "DossierTemplateImportService", contextualName = "import-template")
|
||||
public TemplateImportInfo importDossierTemplate(ImportTemplateResult request) {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
@ -0,0 +1,38 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.FileExchangeArchiveReader;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class FileExchangeArchivalService {
|
||||
|
||||
public static final long SIZE_THRESHOLD = 10000000000L; // 10GB
|
||||
FileManagementServiceSettings settings;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Observed(name = "FileExchangeImportService", contextualName = "read-import-archive")
|
||||
public FileExchangeImportModel readFileExchangeArchive(byte[] archive, String userId) {
|
||||
|
||||
FileExchangeArchiveReader fileExchangeArchiveReader = new FileExchangeArchiveReader(userId);
|
||||
try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
|
||||
zipEntryIterator.forEachRemaining(fileExchangeArchiveReader::handleZipEntryData);
|
||||
}
|
||||
return fileExchangeArchiveReader.buildResult();
|
||||
}
|
||||
|
||||
}
|
||||
@ -3,28 +3,17 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
|
||||
@ -32,15 +21,11 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@ -53,16 +38,12 @@ public class FileExchangeExportService {
|
||||
DossierTemplateExportService dossierTemplateExportService;
|
||||
DossierTemplatePersistenceService dossierTemplatePersistenceService;
|
||||
DossierManagementService dossierManagementService;
|
||||
FileStatusManagementService fileStatusManagementService;
|
||||
FileManagementStorageService storageService;
|
||||
DownloadStatusPersistenceService downloadStatusPersistenceService;
|
||||
ManualChangesExportService manualChangesExportService;
|
||||
DictionaryPersistenceService dictionaryPersistenceService;
|
||||
ComponentLogService componentLogService;
|
||||
ObjectMapper mapper;
|
||||
|
||||
RabbitTemplate rabbitTemplate;
|
||||
EntityTypeExportService entityTypeExportService;
|
||||
DossierIdFileIdRequestValidator requestValidator;
|
||||
DossierExportService dossierExportService;
|
||||
|
||||
|
||||
@Observed(name = "FileExchangeExportService", contextualName = "prepare-export")
|
||||
@ -116,7 +97,7 @@ public class FileExchangeExportService {
|
||||
continue;
|
||||
}
|
||||
|
||||
addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
|
||||
dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
|
||||
}
|
||||
|
||||
storeZipFile(downloadJob.getStorageId(), archiver);
|
||||
@ -126,125 +107,12 @@ public class FileExchangeExportService {
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
|
||||
public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
|
||||
|
||||
List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
|
||||
|
||||
if (!request.dossierIds().contains(dossier.getId()) //
|
||||
&& files.stream()
|
||||
.noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
|
||||
// dossier has no files in requested files and dossier not explicitly requested -> don't export it.
|
||||
return;
|
||||
}
|
||||
|
||||
Path dossierFolder = folder.resolve(dossier.getId());
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
|
||||
|
||||
for (FileModel fileEntity : files) {
|
||||
if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
|
||||
continue;
|
||||
}
|
||||
addFileToArchive(archiver, dossierFolder, request, fileEntity);
|
||||
}
|
||||
|
||||
List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
|
||||
|
||||
String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
|
||||
|
||||
for (TypeEntity type : types) {
|
||||
entityTypeExportService.addEntityTypeToArchive(archiver, type, entitiesFolder);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Observed(name = "FileExchangeExportService", contextualName = "export-file")
|
||||
public void addFileToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, FileModel file) {
|
||||
|
||||
Path fileFolder = folder.resolve(file.getId());
|
||||
|
||||
if (!request.excludeAnalysisLogs()) {
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
|
||||
mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
|
||||
if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
|
||||
mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
|
||||
mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
|
||||
}
|
||||
}
|
||||
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, FileExchangeNames.FILE_STATUS), mapper.writeValueAsBytes(file)));
|
||||
|
||||
if (!request.excludeLayoutFiles()) {
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.STRUCTURE);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TEXT);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
|
||||
}
|
||||
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.HIGHLIGHTS);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TABLES);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMAGES);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VISUAL_LAYOUT);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMPORTED);
|
||||
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.ORIGIN);
|
||||
if (!request.originFileOnly()) {
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.UNTOUCHED);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VIEWER);
|
||||
}
|
||||
|
||||
if (!request.excludeFileAttributes()) {
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.FILE_ATTRIBUTES),
|
||||
mapper.writeValueAsBytes(file.getFileAttributes())));
|
||||
}
|
||||
|
||||
if (!request.excludeManualRedactions()) {
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.MANUAL_REDACTIONS),
|
||||
mapper.writeValueAsBytes(manualChangesExportService.export(file.getId()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static String buildFileName(FileModel file, String name) {
|
||||
|
||||
return buildFileName(file.getId(), name);
|
||||
}
|
||||
|
||||
|
||||
public static String buildFileName(String fileId, String name) {
|
||||
|
||||
return fileId + "." + name;
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void addArchiveModelForStorageFile(FileSystemBackedArchiver archiver, FileModel file, Path fileFolder, FileExchangeNames.Definition definition) {
|
||||
|
||||
String storageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), definition.fileType());
|
||||
if (!storageService.objectExists(storageId)) {
|
||||
log.debug("File {} not found in storage with id {}, skipping...", definition.fileType(), storageId);
|
||||
return;
|
||||
}
|
||||
log.debug("Adding file {} from storage with id {} to archive.", definition.fileType(), storageId);
|
||||
|
||||
byte[] data = storageService.getObject(TenantContext.getTenantId(), storageId).readAllBytes();
|
||||
FileSystemBackedArchiver.ArchiveModel archiveModel = new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, definition.getName()), data);
|
||||
archiver.addEntry(archiveModel);
|
||||
}
|
||||
|
||||
|
||||
@Observed(name = "FileExchangeExportService", contextualName = "store-archive")
|
||||
private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) {
|
||||
|
||||
|
||||
@ -1,37 +1,22 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.FileExchangeArchiveReader;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
|
||||
|
||||
import io.micrometer.common.KeyValue;
|
||||
import io.micrometer.observation.ObservationRegistry;
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@ -41,15 +26,9 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class FileExchangeImportService {
|
||||
|
||||
public static final long SIZE_THRESHOLD = 10000000000L; // 10GB
|
||||
|
||||
FileManagementServiceSettings settings;
|
||||
DossierTemplateImportService dossierTemplateImportService;
|
||||
EntityTypeImportService entityTypeImportService;
|
||||
DossierCreatorService dossierCreatorService;
|
||||
FileImportService fileImportService;
|
||||
FileManagementStorageService storageService;
|
||||
ComponentLogService componentLogService;
|
||||
DossierImportService dossierImportService;
|
||||
FileExchangeArchivalService fileExchangeArchivalService;
|
||||
ReportTemplatePersistenceService reportTemplateService;
|
||||
ObservationRegistry registry;
|
||||
|
||||
@ -59,7 +38,7 @@ public class FileExchangeImportService {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
log.info("Starting file import for user {}", userId);
|
||||
FileExchangeImportModel fileExchangeImportModel = readFileExchangeArchive(archive, userId);
|
||||
FileExchangeImportModel fileExchangeImportModel = fileExchangeArchivalService.readFileExchangeArchive(archive, userId);
|
||||
log.info("Parsed file import archive with {} dossiers and {} files",
|
||||
fileExchangeImportModel.getDossiers().size(),
|
||||
fileExchangeImportModel.getFiles().values()
|
||||
@ -74,6 +53,28 @@ public class FileExchangeImportService {
|
||||
}
|
||||
|
||||
|
||||
private String importFileExchangeModel(FileExchangeImportModel fileExchangeImportModel, String userId) {
|
||||
|
||||
ImportTemplateResult importTemplateResult = fileExchangeImportModel.getImportTemplateResult();
|
||||
|
||||
if (importTemplateResult == null) {
|
||||
throw new BadRequestException("DossierTemplate not present in import archive.");
|
||||
}
|
||||
|
||||
TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importTemplateResult);
|
||||
|
||||
List<String> allReportTemplateIds = reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
|
||||
.stream()
|
||||
.map(ReportTemplateEntity::getTemplateId)
|
||||
.toList();
|
||||
|
||||
for (Dossier dossierToImport : fileExchangeImportModel.getDossiers()) {
|
||||
dossierImportService.importDossier(fileExchangeImportModel, userId, dossierToImport, templateImportInfo, allReportTemplateIds);
|
||||
}
|
||||
return templateImportInfo.getDossierTemplateId();
|
||||
}
|
||||
|
||||
|
||||
private void enrichObservation(String userId, byte[] archive, FileExchangeImportModel importModel) {
|
||||
|
||||
if (registry.getCurrentObservation() != null) {
|
||||
@ -89,107 +90,4 @@ public class FileExchangeImportService {
|
||||
}
|
||||
|
||||
|
||||
@Observed(name = "FileExchangeImportService", contextualName = "write-import-data-to-storage")
|
||||
private String importFileExchangeModel(FileExchangeImportModel fileExchangeImportModel, String userId) {
|
||||
|
||||
TemplateImportInfo templateImportInfo = getDossierTemplateForImport(fileExchangeImportModel);
|
||||
|
||||
List<String> allReportTemplateIds = reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
|
||||
.stream()
|
||||
.map(ReportTemplateEntity::getTemplateId)
|
||||
.toList();
|
||||
|
||||
for (Dossier dossierToImport : fileExchangeImportModel.getDossiers()) {
|
||||
String dossierId = getDossierForImport(templateImportInfo, dossierToImport, userId, allReportTemplateIds);
|
||||
|
||||
entityTypeImportService.importEntityTypes(templateImportInfo.getDossierTemplateId(),
|
||||
dossierId,
|
||||
fileExchangeImportModel.getDossierTypes()
|
||||
.get(dossierToImport.getId()));
|
||||
|
||||
List<FileExchangeImportModel.FileImport> files = fileExchangeImportModel.getFiles().getOrDefault(dossierToImport.getId(), Collections.emptyList());
|
||||
|
||||
for (FileExchangeImportModel.FileImport file : files) {
|
||||
// separate service for transactions
|
||||
String fileId = fileImportService.saveFileToDb(userId, file, dossierId, templateImportInfo);
|
||||
|
||||
file.getFilesToUpload()
|
||||
.forEach(fileToUpload -> {
|
||||
try (FileInputStream in = new FileInputStream(fileToUpload.file().toFile())) {
|
||||
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileToUpload.fileType()), in);
|
||||
Files.deleteIfExists(fileToUpload.file());
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
storageService.saveEntityLog(dossierId, fileId, file.getEntityLog());
|
||||
if (file.getComponentLog() != null) {
|
||||
storageService.saveComponentLog(dossierId, fileId, file.getComponentLog());
|
||||
}
|
||||
if (file.getOverrides() != null) {
|
||||
file.getOverrides()
|
||||
.forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return templateImportInfo.getDossierTemplateId();
|
||||
}
|
||||
|
||||
|
||||
private String getDossierForImport(TemplateImportInfo templateImportInfo, Dossier dossier, String userId, List<String> reportTemplateIds) {
|
||||
|
||||
String dossierTemplateId = templateImportInfo.getDossierTemplateId();
|
||||
|
||||
CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
|
||||
.dossierName(dossier.getName())
|
||||
.description(dossier.getDescription())
|
||||
.downloadFileTypes(dossier.getDownloadFileTypes())
|
||||
.dueDate(dossier.getDueDate())
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.requestingUser(userId)
|
||||
.reportTemplateIds(reportTemplateIds)
|
||||
.build();
|
||||
|
||||
Dossier importedDossier = null;
|
||||
int retries = 0;
|
||||
while (importedDossier == null && retries < 10) {
|
||||
try {
|
||||
importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
|
||||
} catch (ConflictException e) {
|
||||
retries++;
|
||||
request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
|
||||
}
|
||||
}
|
||||
if (importedDossier == null) {
|
||||
throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
|
||||
}
|
||||
return importedDossier.getId();
|
||||
}
|
||||
|
||||
|
||||
private TemplateImportInfo getDossierTemplateForImport(FileExchangeImportModel fileExchangeImportModel) {
|
||||
|
||||
ImportTemplateResult importTemplateResult = fileExchangeImportModel.getImportTemplateResult();
|
||||
|
||||
if (importTemplateResult == null) {
|
||||
throw new BadRequestException("DossierTemplate not present in import archive.");
|
||||
}
|
||||
|
||||
return dossierTemplateImportService.importDossierTemplate(importTemplateResult);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Observed(name = "FileExchangeImportService", contextualName = "read-import-archive")
|
||||
private FileExchangeImportModel readFileExchangeArchive(byte[] archive, String userId) {
|
||||
|
||||
FileExchangeArchiveReader fileExchangeArchiveReader = new FileExchangeArchiveReader(userId);
|
||||
try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
|
||||
zipEntryIterator.forEachRemaining(fileExchangeArchiveReader::handleZipEntryData);
|
||||
}
|
||||
return fileExchangeArchiveReader.buildResult();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,122 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class FileExportService {
|
||||
|
||||
ManualChangesExportService manualChangesExportService;
|
||||
ComponentLogService componentLogService;
|
||||
ObjectMapper mapper;
|
||||
FileManagementStorageService storageService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Observed(name = "FileExchangeExportService", contextualName = "export-file")
|
||||
public void addFileToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, FileModel file) {
|
||||
|
||||
Path fileFolder = folder.resolve(file.getId());
|
||||
|
||||
if (!request.excludeAnalysisLogs()) {
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
|
||||
mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
|
||||
if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
|
||||
mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
|
||||
mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
|
||||
}
|
||||
}
|
||||
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, FileExchangeNames.FILE_STATUS), mapper.writeValueAsBytes(file)));
|
||||
|
||||
if (!request.excludeLayoutFiles()) {
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.STRUCTURE);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TEXT);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
|
||||
}
|
||||
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.HIGHLIGHTS);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TABLES);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMAGES);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VISUAL_LAYOUT);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMPORTED);
|
||||
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.ORIGIN);
|
||||
if (!request.originFileOnly()) {
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.UNTOUCHED);
|
||||
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VIEWER);
|
||||
}
|
||||
|
||||
if (!request.excludeFileAttributes()) {
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.FILE_ATTRIBUTES),
|
||||
mapper.writeValueAsBytes(file.getFileAttributes())));
|
||||
}
|
||||
|
||||
if (!request.excludeManualRedactions()) {
|
||||
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
|
||||
buildFileName(file, FileExchangeNames.MANUAL_REDACTIONS),
|
||||
mapper.writeValueAsBytes(manualChangesExportService.export(file.getId()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static String buildFileName(FileModel file, String name) {
|
||||
|
||||
return buildFileName(file.getId(), name);
|
||||
}
|
||||
|
||||
|
||||
public static String buildFileName(String fileId, String name) {
|
||||
|
||||
return fileId + "." + name;
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void addArchiveModelForStorageFile(FileSystemBackedArchiver archiver, FileModel file, Path fileFolder, FileExchangeNames.Definition definition) {
|
||||
|
||||
String storageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), definition.fileType());
|
||||
if (!storageService.objectExists(storageId)) {
|
||||
log.debug("File {} not found in storage with id {}, skipping...", definition.fileType(), storageId);
|
||||
return;
|
||||
}
|
||||
log.debug("Adding file {} from storage with id {} to archive.", definition.fileType(), storageId);
|
||||
|
||||
byte[] data = storageService.getObject(TenantContext.getTenantId(), storageId).readAllBytes();
|
||||
FileSystemBackedArchiver.ArchiveModel archiveModel = new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, definition.getName()), data);
|
||||
archiver.addEntry(archiveModel);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.FileEntityMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class FileImportPersistenceService {
|
||||
|
||||
|
||||
ManualChangesImportService manualChangesImportService;
|
||||
FileRepository fileRepository;
|
||||
|
||||
|
||||
@Transactional
|
||||
@Observed(name = "FileImportPersistenceService", contextualName = "import-file-to-db")
|
||||
public synchronized String saveFileToDb(String userId, FileExchangeImportModel.FileImport file, String dossierId, TemplateImportInfo templateImportInfo) { // synchronized as this is being called in an async block. Might lock up DB otherwise.
|
||||
|
||||
String fileId = createFile(dossierId, userId, file, templateImportInfo.getIdMapping());
|
||||
|
||||
manualChangesImportService.importManualChanges(file.getManualChanges(), fileId);
|
||||
|
||||
return fileId;
|
||||
}
|
||||
|
||||
|
||||
private String createFile(String dossierId, String userid, FileExchangeImportModel.FileImport file, Map<String, String> fileAttributeConfigMap) {
|
||||
|
||||
FileModel fileModel = file.getFileModel();
|
||||
|
||||
fileModel.setId(UploadService.generateFileId(fileModel.getFilename(), dossierId));
|
||||
|
||||
FileEntity fileEntity = MagicConverter.convert(fileModel, FileEntity.class, new FileEntityMapper(fileAttributeConfigMap));
|
||||
|
||||
fileEntity.setDossierId(dossierId);
|
||||
|
||||
fileRepository.save(fileEntity);
|
||||
|
||||
return fileEntity.getId();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,57 +1,78 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
|
||||
|
||||
import java.util.Map;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.FileEntityMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import jakarta.transaction.Transactional;
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class FileImportService {
|
||||
|
||||
|
||||
ManualChangesImportService manualChangesImportService;
|
||||
FileRepository fileRepository;
|
||||
FileImportPersistenceService fileImportPersistenceService;
|
||||
FileManagementStorageService storageService;
|
||||
ComponentLogService componentLogService;
|
||||
|
||||
|
||||
@Transactional
|
||||
public String saveFileToDb(String userId, FileExchangeImportModel.FileImport file, String dossierId, TemplateImportInfo templateImportInfo) {
|
||||
@Observed(name = "FileImportService", contextualName = "import-files-in-parallel")
|
||||
public void importFilesInParallel(List<FileExchangeImportModel.FileImport> files, String userId, TemplateImportInfo templateImportInfo, String dossierId) {
|
||||
|
||||
String fileId = createFile(dossierId, userId, file, templateImportInfo.getIdMapping());
|
||||
List<CompletableFuture<Void>> fileImportFutures = new LinkedList<>();
|
||||
|
||||
manualChangesImportService.importManualChanges(file.getManualChanges(), fileId);
|
||||
String tenantId = TenantContext.getTenantId();
|
||||
|
||||
return fileId;
|
||||
for (FileExchangeImportModel.FileImport file : files) {
|
||||
fileImportFutures.add(CompletableFuture.supplyAsync(() -> {
|
||||
TenantContext.setTenantId(tenantId);
|
||||
importFile(userId, templateImportInfo, file, dossierId);
|
||||
return null;
|
||||
}));
|
||||
}
|
||||
fileImportFutures.forEach(CompletableFuture::join);
|
||||
}
|
||||
|
||||
|
||||
private String createFile(String dossierId, String userid, FileExchangeImportModel.FileImport file, Map<String, String> fileAttributeConfigMap) {
|
||||
private void importFile(String userId, TemplateImportInfo templateImportInfo, FileExchangeImportModel.FileImport file, String dossierId) {
|
||||
|
||||
FileModel fileModel = file.getFileModel();
|
||||
// separate service for transactions
|
||||
String fileId = fileImportPersistenceService.saveFileToDb(userId, file, dossierId, templateImportInfo);
|
||||
|
||||
fileModel.setId(UploadService.generateFileId(fileModel.getFilename(), dossierId));
|
||||
file.getFilesToUpload()
|
||||
.forEach(fileToUpload -> {
|
||||
try (FileInputStream in = new FileInputStream(fileToUpload.file().toFile())) {
|
||||
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileToUpload.fileType()), in);
|
||||
Files.deleteIfExists(fileToUpload.file());
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
FileEntity fileEntity = MagicConverter.convert(fileModel, FileEntity.class, new FileEntityMapper(fileAttributeConfigMap));
|
||||
|
||||
fileEntity.setDossierId(dossierId);
|
||||
|
||||
fileRepository.save(fileEntity);
|
||||
|
||||
return fileEntity.getId();
|
||||
storageService.saveEntityLog(dossierId, fileId, file.getEntityLog());
|
||||
if (file.getComponentLog() != null) {
|
||||
storageService.saveComponentLog(dossierId, fileId, file.getComponentLog());
|
||||
}
|
||||
if (file.getOverrides() != null) {
|
||||
file.getOverrides()
|
||||
.forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -33,6 +34,7 @@ public class ManualChangesExportService {
|
||||
|
||||
|
||||
@Transactional
|
||||
@Observed(name = "ManualChangesExportService", contextualName = "export-manual-changes")
|
||||
public ManualChangesExportModel export(String fileId) {
|
||||
|
||||
ManualChangesExportModel exportModel = new ManualChangesExportModel();
|
||||
|
||||
@ -13,6 +13,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -36,6 +37,7 @@ public class ManualChangesImportService {
|
||||
|
||||
|
||||
@Transactional
|
||||
@Observed(name = "ManualChangesImportService", contextualName = "import-manual-changes")
|
||||
public void importManualChanges(ManualChangesExportModel exportModel, String fileId) {
|
||||
|
||||
FileEntity fileEntity = fileRepository.findById(fileId)
|
||||
|
||||
@ -7,19 +7,17 @@ import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.dao.DataIntegrityViolationException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ComponentMappingRepository;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@ -6,6 +6,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
|
||||
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -24,6 +25,7 @@ public class DossierCreatorService {
|
||||
|
||||
|
||||
@Transactional
|
||||
@Observed(name = "DossierCreatorService", contextualName = "create-dossier-acl")
|
||||
public Dossier addDossier(CreateOrUpdateDossierRequest dossierRequest, Set<String> members, Set<String> approvers, String ownerId) {
|
||||
|
||||
var dossier = dossierService.addDossier(dossierRequest);
|
||||
|
||||
@ -6,6 +6,7 @@ import java.util.Optional;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DigitalSignatureEntity;
|
||||
@ -22,7 +23,7 @@ public interface ComponentMappingRepository extends JpaRepository<ComponentMappi
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update ComponentMappingEntity ce set ce.version = :version where ce.id = :typeId")
|
||||
void updateVersion(String componentMappingId, Integer version);
|
||||
@Query("update ComponentMappingEntity ce set ce.version = :version where ce.id = :componentMappingId")
|
||||
void updateVersion(@Param("componentMappingId") String componentMappingId,@Param("version") Integer version);
|
||||
|
||||
}
|
||||
|
||||
@ -24,10 +24,8 @@ dependencies {
|
||||
api(project(":persistence-service-internal-api-impl-v1"))
|
||||
api(project(":persistence-service-shared-mongo-v1"))
|
||||
api("com.iqser.red.commons:storage-commons:2.49.0")
|
||||
api("junit:junit:4.13.2")
|
||||
api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
|
||||
api("net.logstash.logback:logstash-logback-encoder:7.4")
|
||||
|
||||
testImplementation("junit:junit:4.13.2")
|
||||
testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
|
||||
testImplementation("org.springframework.security:spring-security-test:6.0.2")
|
||||
testImplementation("org.testcontainers:postgresql:1.17.1")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user