RED-9255: fix annotations, refactor for observability #562

Merged
kilian.schuettler1 merged 1 commits from RED-9255 into master 2024-06-28 09:55:34 +02:00
15 changed files with 466 additions and 300 deletions

View File

@ -0,0 +1,70 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import java.nio.file.Path;
import java.util.List;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DossierExportService {
DictionaryPersistenceService dictionaryPersistenceService;
FileExportService fileExportService;
FileStatusManagementService fileStatusManagementService;
EntityTypeExportService entityTypeExportService;
ObjectMapper mapper;
@SneakyThrows
@Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
if (!request.dossierIds().contains(dossier.getId()) //
&& files.stream()
.noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
// dossier has no files in requested files and dossier not explicitly requested -> don't export it.
return;
}
Path dossierFolder = folder.resolve(dossier.getId());
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
for (FileModel fileEntity : files) {
if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
continue;
}
fileExportService.addFileToArchive(archiver, dossierFolder, request, fileEntity);
}
List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
for (TypeEntity type : types) {
entityTypeExportService.addEntityTypeToArchive(archiver, type, entitiesFolder);
}
}
}

View File

@ -0,0 +1,86 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DossierImportService {
DossierCreatorService dossierCreatorService;
FileImportService fileImportService;
EntityTypeImportService entityTypeImportService;
ObservationRegistry registry;
@Observed(name = "DossierImportService", contextualName = "import-dossier")
public void importDossier(FileExchangeImportModel fileExchangeImportModel,
String userId,
Dossier dossierToImport,
TemplateImportInfo templateImportInfo,
List<String> allReportTemplateIds) {
String dossierId = getDossierForImport(templateImportInfo, dossierToImport, userId, allReportTemplateIds);
entityTypeImportService.importEntityTypes(templateImportInfo.getDossierTemplateId(),
dossierId,
fileExchangeImportModel.getDossierTypes()
.get(dossierToImport.getId()));
List<FileExchangeImportModel.FileImport> files = fileExchangeImportModel.getFiles().getOrDefault(dossierToImport.getId(), Collections.emptyList());
fileImportService.importFilesInParallel(files, userId, templateImportInfo, dossierId);
}
private String getDossierForImport(TemplateImportInfo templateImportInfo, Dossier dossier, String userId, List<String> reportTemplateIds) {
String dossierTemplateId = templateImportInfo.getDossierTemplateId();
CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
.dossierName(dossier.getName())
.description(dossier.getDescription())
.downloadFileTypes(dossier.getDownloadFileTypes())
.dueDate(dossier.getDueDate())
.dossierTemplateId(dossierTemplateId)
.requestingUser(userId)
.reportTemplateIds(reportTemplateIds)
.build();
Dossier importedDossier = null;
int retries = 0;
while (importedDossier == null && retries < 10) {
try {
importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
} catch (ConflictException e) {
retries++;
request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
}
}
if (importedDossier == null) {
throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
}
return importedDossier.getId();
}
}

View File

@ -67,6 +67,7 @@ import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@ -120,7 +121,7 @@ public class DossierTemplateImportService {
return dossierTemplateArchiveReader.buildResult();
}
@Observed(name = "DossierTemplateImportService", contextualName = "import-template")
public TemplateImportInfo importDossierTemplate(ImportTemplateResult request) {
long start = System.currentTimeMillis();

View File

@ -0,0 +1,38 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.FileExchangeArchiveReader;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class FileExchangeArchivalService {
public static final long SIZE_THRESHOLD = 10000000000L; // 10GB
FileManagementServiceSettings settings;
@SneakyThrows
@Observed(name = "FileExchangeImportService", contextualName = "read-import-archive")
public FileExchangeImportModel readFileExchangeArchive(byte[] archive, String userId) {
FileExchangeArchiveReader fileExchangeArchiveReader = new FileExchangeArchiveReader(userId);
try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
zipEntryIterator.forEachRemaining(fileExchangeArchiveReader::handleZipEntryData);
}
return fileExchangeArchiveReader.buildResult();
}
}

View File

@ -3,28 +3,17 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
@ -32,15 +21,11 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId
import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@ -53,16 +38,12 @@ public class FileExchangeExportService {
DossierTemplateExportService dossierTemplateExportService;
DossierTemplatePersistenceService dossierTemplatePersistenceService;
DossierManagementService dossierManagementService;
FileStatusManagementService fileStatusManagementService;
FileManagementStorageService storageService;
DownloadStatusPersistenceService downloadStatusPersistenceService;
ManualChangesExportService manualChangesExportService;
DictionaryPersistenceService dictionaryPersistenceService;
ComponentLogService componentLogService;
ObjectMapper mapper;
RabbitTemplate rabbitTemplate;
EntityTypeExportService entityTypeExportService;
DossierIdFileIdRequestValidator requestValidator;
DossierExportService dossierExportService;
@Observed(name = "FileExchangeExportService", contextualName = "prepare-export")
@ -116,7 +97,7 @@ public class FileExchangeExportService {
continue;
}
addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
}
storeZipFile(downloadJob.getStorageId(), archiver);
@ -126,125 +107,12 @@ public class FileExchangeExportService {
}
@SneakyThrows
@Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
if (!request.dossierIds().contains(dossier.getId()) //
&& files.stream()
.noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
// dossier has no files in requested files and dossier not explicitly requested -> don't export it.
return;
}
Path dossierFolder = folder.resolve(dossier.getId());
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
for (FileModel fileEntity : files) {
if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
continue;
}
addFileToArchive(archiver, dossierFolder, request, fileEntity);
}
List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
for (TypeEntity type : types) {
entityTypeExportService.addEntityTypeToArchive(archiver, type, entitiesFolder);
}
}
@SneakyThrows
@Observed(name = "FileExchangeExportService", contextualName = "export-file")
public void addFileToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, FileModel file) {
Path fileFolder = folder.resolve(file.getId());
if (!request.excludeAnalysisLogs()) {
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
}
}
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, FileExchangeNames.FILE_STATUS), mapper.writeValueAsBytes(file)));
if (!request.excludeLayoutFiles()) {
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.STRUCTURE);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TEXT);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
}
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.HIGHLIGHTS);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TABLES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMAGES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VISUAL_LAYOUT);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMPORTED);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.ORIGIN);
if (!request.originFileOnly()) {
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.UNTOUCHED);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VIEWER);
}
if (!request.excludeFileAttributes()) {
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.FILE_ATTRIBUTES),
mapper.writeValueAsBytes(file.getFileAttributes())));
}
if (!request.excludeManualRedactions()) {
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.MANUAL_REDACTIONS),
mapper.writeValueAsBytes(manualChangesExportService.export(file.getId()))));
}
}
private static String buildFileName(FileModel file, String name) {
return buildFileName(file.getId(), name);
}
public static String buildFileName(String fileId, String name) {
return fileId + "." + name;
}
@SneakyThrows
private void addArchiveModelForStorageFile(FileSystemBackedArchiver archiver, FileModel file, Path fileFolder, FileExchangeNames.Definition definition) {
String storageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), definition.fileType());
if (!storageService.objectExists(storageId)) {
log.debug("File {} not found in storage with id {}, skipping...", definition.fileType(), storageId);
return;
}
log.debug("Adding file {} from storage with id {} to archive.", definition.fileType(), storageId);
byte[] data = storageService.getObject(TenantContext.getTenantId(), storageId).readAllBytes();
FileSystemBackedArchiver.ArchiveModel archiveModel = new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, definition.getName()), data);
archiver.addEntry(archiveModel);
}
@Observed(name = "FileExchangeExportService", contextualName = "store-archive")
private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) {

View File

@ -1,37 +1,22 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import java.io.FileInputStream;
import java.nio.file.Files;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.FileExchangeArchiveReader;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import io.micrometer.common.KeyValue;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@ -41,15 +26,9 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class FileExchangeImportService {
public static final long SIZE_THRESHOLD = 10000000000L; // 10GB
FileManagementServiceSettings settings;
DossierTemplateImportService dossierTemplateImportService;
EntityTypeImportService entityTypeImportService;
DossierCreatorService dossierCreatorService;
FileImportService fileImportService;
FileManagementStorageService storageService;
ComponentLogService componentLogService;
DossierImportService dossierImportService;
FileExchangeArchivalService fileExchangeArchivalService;
ReportTemplatePersistenceService reportTemplateService;
ObservationRegistry registry;
@ -59,7 +38,7 @@ public class FileExchangeImportService {
long start = System.currentTimeMillis();
log.info("Starting file import for user {}", userId);
FileExchangeImportModel fileExchangeImportModel = readFileExchangeArchive(archive, userId);
FileExchangeImportModel fileExchangeImportModel = fileExchangeArchivalService.readFileExchangeArchive(archive, userId);
log.info("Parsed file import archive with {} dossiers and {} files",
fileExchangeImportModel.getDossiers().size(),
fileExchangeImportModel.getFiles().values()
@ -74,6 +53,28 @@ public class FileExchangeImportService {
}
private String importFileExchangeModel(FileExchangeImportModel fileExchangeImportModel, String userId) {
ImportTemplateResult importTemplateResult = fileExchangeImportModel.getImportTemplateResult();
if (importTemplateResult == null) {
throw new BadRequestException("DossierTemplate not present in import archive.");
}
TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importTemplateResult);
List<String> allReportTemplateIds = reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
.stream()
.map(ReportTemplateEntity::getTemplateId)
.toList();
for (Dossier dossierToImport : fileExchangeImportModel.getDossiers()) {
dossierImportService.importDossier(fileExchangeImportModel, userId, dossierToImport, templateImportInfo, allReportTemplateIds);
}
return templateImportInfo.getDossierTemplateId();
}
private void enrichObservation(String userId, byte[] archive, FileExchangeImportModel importModel) {
if (registry.getCurrentObservation() != null) {
@ -89,107 +90,4 @@ public class FileExchangeImportService {
}
@Observed(name = "FileExchangeImportService", contextualName = "write-import-data-to-storage")
private String importFileExchangeModel(FileExchangeImportModel fileExchangeImportModel, String userId) {
TemplateImportInfo templateImportInfo = getDossierTemplateForImport(fileExchangeImportModel);
List<String> allReportTemplateIds = reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
.stream()
.map(ReportTemplateEntity::getTemplateId)
.toList();
for (Dossier dossierToImport : fileExchangeImportModel.getDossiers()) {
String dossierId = getDossierForImport(templateImportInfo, dossierToImport, userId, allReportTemplateIds);
entityTypeImportService.importEntityTypes(templateImportInfo.getDossierTemplateId(),
dossierId,
fileExchangeImportModel.getDossierTypes()
.get(dossierToImport.getId()));
List<FileExchangeImportModel.FileImport> files = fileExchangeImportModel.getFiles().getOrDefault(dossierToImport.getId(), Collections.emptyList());
for (FileExchangeImportModel.FileImport file : files) {
// separate service for transactions
String fileId = fileImportService.saveFileToDb(userId, file, dossierId, templateImportInfo);
file.getFilesToUpload()
.forEach(fileToUpload -> {
try (FileInputStream in = new FileInputStream(fileToUpload.file().toFile())) {
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileToUpload.fileType()), in);
Files.deleteIfExists(fileToUpload.file());
} catch (Exception e) {
throw new RuntimeException(e);
}
});
storageService.saveEntityLog(dossierId, fileId, file.getEntityLog());
if (file.getComponentLog() != null) {
storageService.saveComponentLog(dossierId, fileId, file.getComponentLog());
}
if (file.getOverrides() != null) {
file.getOverrides()
.forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
}
}
}
return templateImportInfo.getDossierTemplateId();
}
private String getDossierForImport(TemplateImportInfo templateImportInfo, Dossier dossier, String userId, List<String> reportTemplateIds) {
String dossierTemplateId = templateImportInfo.getDossierTemplateId();
CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
.dossierName(dossier.getName())
.description(dossier.getDescription())
.downloadFileTypes(dossier.getDownloadFileTypes())
.dueDate(dossier.getDueDate())
.dossierTemplateId(dossierTemplateId)
.requestingUser(userId)
.reportTemplateIds(reportTemplateIds)
.build();
Dossier importedDossier = null;
int retries = 0;
while (importedDossier == null && retries < 10) {
try {
importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
} catch (ConflictException e) {
retries++;
request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
}
}
if (importedDossier == null) {
throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
}
return importedDossier.getId();
}
private TemplateImportInfo getDossierTemplateForImport(FileExchangeImportModel fileExchangeImportModel) {
ImportTemplateResult importTemplateResult = fileExchangeImportModel.getImportTemplateResult();
if (importTemplateResult == null) {
throw new BadRequestException("DossierTemplate not present in import archive.");
}
return dossierTemplateImportService.importDossierTemplate(importTemplateResult);
}
@SneakyThrows
@Observed(name = "FileExchangeImportService", contextualName = "read-import-archive")
private FileExchangeImportModel readFileExchangeArchive(byte[] archive, String userId) {
FileExchangeArchiveReader fileExchangeArchiveReader = new FileExchangeArchiveReader(userId);
try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
zipEntryIterator.forEachRemaining(fileExchangeArchiveReader::handleZipEntryData);
}
return fileExchangeArchiveReader.buildResult();
}
}

View File

@ -0,0 +1,122 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import java.nio.file.Path;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class FileExportService {
ManualChangesExportService manualChangesExportService;
ComponentLogService componentLogService;
ObjectMapper mapper;
FileManagementStorageService storageService;
@SneakyThrows
@Observed(name = "FileExchangeExportService", contextualName = "export-file")
public void addFileToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, FileModel file) {
Path fileFolder = folder.resolve(file.getId());
if (!request.excludeAnalysisLogs()) {
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
}
}
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, FileExchangeNames.FILE_STATUS), mapper.writeValueAsBytes(file)));
if (!request.excludeLayoutFiles()) {
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.STRUCTURE);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TEXT);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
}
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.HIGHLIGHTS);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TABLES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMAGES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VISUAL_LAYOUT);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMPORTED);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.ORIGIN);
if (!request.originFileOnly()) {
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.UNTOUCHED);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VIEWER);
}
if (!request.excludeFileAttributes()) {
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.FILE_ATTRIBUTES),
mapper.writeValueAsBytes(file.getFileAttributes())));
}
if (!request.excludeManualRedactions()) {
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
buildFileName(file, FileExchangeNames.MANUAL_REDACTIONS),
mapper.writeValueAsBytes(manualChangesExportService.export(file.getId()))));
}
}
private static String buildFileName(FileModel file, String name) {
return buildFileName(file.getId(), name);
}
public static String buildFileName(String fileId, String name) {
return fileId + "." + name;
}
@SneakyThrows
private void addArchiveModelForStorageFile(FileSystemBackedArchiver archiver, FileModel file, Path fileFolder, FileExchangeNames.Definition definition) {
String storageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), definition.fileType());
if (!storageService.objectExists(storageId)) {
log.debug("File {} not found in storage with id {}, skipping...", definition.fileType(), storageId);
return;
}
log.debug("Adding file {} from storage with id {} to archive.", definition.fileType(), storageId);
byte[] data = storageService.getObject(TenantContext.getTenantId(), storageId).readAllBytes();
FileSystemBackedArchiver.ArchiveModel archiveModel = new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, definition.getName()), data);
archiver.addEntry(archiveModel);
}
}

View File

@ -0,0 +1,59 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import java.util.Map;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileEntityMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import io.micrometer.observation.annotation.Observed;
import jakarta.transaction.Transactional;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class FileImportPersistenceService {
ManualChangesImportService manualChangesImportService;
FileRepository fileRepository;
@Transactional
@Observed(name = "FileImportPersistenceService", contextualName = "import-file-to-db")
public synchronized String saveFileToDb(String userId, FileExchangeImportModel.FileImport file, String dossierId, TemplateImportInfo templateImportInfo) { // synchronized as this is being called in an async block. Might lock up DB otherwise.
String fileId = createFile(dossierId, userId, file, templateImportInfo.getIdMapping());
manualChangesImportService.importManualChanges(file.getManualChanges(), fileId);
return fileId;
}
private String createFile(String dossierId, String userid, FileExchangeImportModel.FileImport file, Map<String, String> fileAttributeConfigMap) {
FileModel fileModel = file.getFileModel();
fileModel.setId(UploadService.generateFileId(fileModel.getFilename(), dossierId));
FileEntity fileEntity = MagicConverter.convert(fileModel, FileEntity.class, new FileEntityMapper(fileAttributeConfigMap));
fileEntity.setDossierId(dossierId);
fileRepository.save(fileEntity);
return fileEntity.getId();
}
}

View File

@ -1,57 +1,78 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import java.util.Map;
import java.io.FileInputStream;
import java.nio.file.Files;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileEntityMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.knecon.fforesight.tenantcommons.TenantContext;
import jakarta.transaction.Transactional;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class FileImportService {
ManualChangesImportService manualChangesImportService;
FileRepository fileRepository;
FileImportPersistenceService fileImportPersistenceService;
FileManagementStorageService storageService;
ComponentLogService componentLogService;
@Transactional
public String saveFileToDb(String userId, FileExchangeImportModel.FileImport file, String dossierId, TemplateImportInfo templateImportInfo) {
@Observed(name = "FileImportService", contextualName = "import-files-in-parallel")
public void importFilesInParallel(List<FileExchangeImportModel.FileImport> files, String userId, TemplateImportInfo templateImportInfo, String dossierId) {
String fileId = createFile(dossierId, userId, file, templateImportInfo.getIdMapping());
List<CompletableFuture<Void>> fileImportFutures = new LinkedList<>();
manualChangesImportService.importManualChanges(file.getManualChanges(), fileId);
String tenantId = TenantContext.getTenantId();
return fileId;
for (FileExchangeImportModel.FileImport file : files) {
fileImportFutures.add(CompletableFuture.supplyAsync(() -> {
TenantContext.setTenantId(tenantId);
importFile(userId, templateImportInfo, file, dossierId);
return null;
}));
}
fileImportFutures.forEach(CompletableFuture::join);
}
private String createFile(String dossierId, String userid, FileExchangeImportModel.FileImport file, Map<String, String> fileAttributeConfigMap) {
private void importFile(String userId, TemplateImportInfo templateImportInfo, FileExchangeImportModel.FileImport file, String dossierId) {
FileModel fileModel = file.getFileModel();
// separate service for transactions
String fileId = fileImportPersistenceService.saveFileToDb(userId, file, dossierId, templateImportInfo);
fileModel.setId(UploadService.generateFileId(fileModel.getFilename(), dossierId));
file.getFilesToUpload()
.forEach(fileToUpload -> {
try (FileInputStream in = new FileInputStream(fileToUpload.file().toFile())) {
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileToUpload.fileType()), in);
Files.deleteIfExists(fileToUpload.file());
} catch (Exception e) {
throw new RuntimeException(e);
}
});
FileEntity fileEntity = MagicConverter.convert(fileModel, FileEntity.class, new FileEntityMapper(fileAttributeConfigMap));
fileEntity.setDossierId(dossierId);
fileRepository.save(fileEntity);
return fileEntity.getId();
storageService.saveEntityLog(dossierId, fileId, file.getEntityLog());
if (file.getComponentLog() != null) {
storageService.saveComponentLog(dossierId, fileId, file.getComponentLog());
}
if (file.getOverrides() != null) {
file.getOverrides()
.forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
}
}
}

View File

@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import io.micrometer.observation.annotation.Observed;
import jakarta.transaction.Transactional;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -33,6 +34,7 @@ public class ManualChangesExportService {
@Transactional
@Observed(name = "ManualChangesExportService", contextualName = "export-manual-changes")
public ManualChangesExportModel export(String fileId) {
ManualChangesExportModel exportModel = new ManualChangesExportModel();

View File

@ -13,6 +13,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository;
import io.micrometer.observation.annotation.Observed;
import jakarta.transaction.Transactional;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -36,6 +37,7 @@ public class ManualChangesImportService {
@Transactional
@Observed(name = "ManualChangesImportService", contextualName = "import-manual-changes")
public void importManualChanges(ManualChangesExportModel exportModel, String fileId) {
FileEntity fileEntity = fileRepository.findById(fileId)

View File

@ -7,19 +7,17 @@ import java.time.OffsetDateTime;
import java.util.List;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ComponentMappingRepository;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import jakarta.validation.ConstraintViolationException;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;

View File

@ -6,6 +6,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import io.micrometer.observation.annotation.Observed;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -24,6 +25,7 @@ public class DossierCreatorService {
@Transactional
@Observed(name = "DossierCreatorService", contextualName = "create-dossier-acl")
public Dossier addDossier(CreateOrUpdateDossierRequest dossierRequest, Set<String> members, Set<String> approvers, String ownerId) {
var dossier = dossierService.addDossier(dossierRequest);

View File

@ -6,6 +6,7 @@ import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DigitalSignatureEntity;
@ -22,7 +23,7 @@ public interface ComponentMappingRepository extends JpaRepository<ComponentMappi
@Modifying
@Query("update ComponentMappingEntity ce set ce.version = :version where ce.id = :typeId")
void updateVersion(String componentMappingId, Integer version);
@Query("update ComponentMappingEntity ce set ce.version = :version where ce.id = :componentMappingId")
void updateVersion(@Param("componentMappingId") String componentMappingId,@Param("version") Integer version);
}

View File

@ -24,10 +24,8 @@ dependencies {
api(project(":persistence-service-internal-api-impl-v1"))
api(project(":persistence-service-shared-mongo-v1"))
api("com.iqser.red.commons:storage-commons:2.49.0")
api("junit:junit:4.13.2")
api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
api("net.logstash.logback:logstash-logback-encoder:7.4")
testImplementation("junit:junit:4.13.2")
testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
testImplementation("org.springframework.security:spring-security-test:6.0.2")
testImplementation("org.testcontainers:postgresql:1.17.1")