Pull request #534: Feature/RED-2200 1

Merge in RED/persistence-service from feature/RED-2200_1 to master

* commit '61e9456a9b463b58f7eaebd937a8ad0ccdecfc0a':
  RED-2200 - Dossier Template export and import
  RED-2200 - Dossier Template export and import
This commit is contained in:
Corina Olariu 2022-09-22 13:58:49 +02:00
commit d34a92c64b
6 changed files with 143 additions and 109 deletions

View File

@ -14,6 +14,7 @@ public enum ExportFilename {
LEGAL_BASIS("legalBasisMappingList"),
RULES("rules"),
REPORT_TEMPLATE("reportTemplateList"),
REPORT_TEMPLATE_MULTI_FILE(" (Multi-file)"),
DOSSIER_TYPE("dossierType"),
ENTRIES("entries"),
FALSE_POSITIVES("falsePositives"),

View File

@ -72,7 +72,8 @@ public class DossierTemplateController implements DossierTemplateResource {
}
public DossierTemplate importDossierTemplate(@RequestBody ImportDossierTemplateRequest request) {
return dossierTemplateImportService.importDossierTemplate(request);
String dossierTemplateId = dossierTemplateImportService.importDossierTemplate(request);
return convert(dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId), DossierTemplate.class);
}
}

View File

@ -218,10 +218,9 @@ public class DictionaryService {
List<TypeEntity> typeResponse = dictionaryPersistenceService.getCumulatedTypes(dossierTemplateId, dossierId, false);
for (TypeEntity res : typeResponse) {
if (res.getDossierId() != null && res.getDossierId().equals(dossierId) && !type.equals(res.getType()) && res
.getDossierTemplateId()
.equals(dossierTemplateId) && labelToCheck.equals(res.getLabel()) || !type.equals(res.getType()) && res
.getDossierTemplateId()
var dossierTemplateResponse = res.getDossierTemplateId() == null ? res.getDossierTemplate().getId() : res.getDossierTemplateId();
if (res.getDossierId() != null && res.getDossierId().equals(dossierId) && !type.equals(res.getType()) && dossierTemplateResponse
.equals(dossierTemplateId) && labelToCheck.equals(res.getLabel()) || !type.equals(res.getType()) && dossierTemplateResponse
.equals(dossierTemplateId) && labelToCheck.equals(res.getLabel())) {
throw new ConflictException("Label must be unique.");
}

View File

@ -3,16 +3,13 @@ package com.iqser.red.service.peristence.v1.server.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
@ -75,15 +72,16 @@ public class DossierTemplateImportService {
private final WatermarkService watermarkService;
private final ReportTemplateService reportTemplateService;
private final DictionaryService dictionaryService;
private final EntryPersistenceService entryPersistenceService;
private final ObjectMapper objectMapper = new ObjectMapper();
public DossierTemplate importDossierTemplate(@RequestBody ImportDossierTemplateRequest request) {
ImportTemplateResult result = this.handleArchive(request);
return importDossierTemplate(result);
public String importDossierTemplate(@RequestBody ImportDossierTemplateRequest request) {
ImportTemplateResult archiveResult = this.handleArchive(request);
return this.importDossierTemplate(archiveResult);
}
@Transactional
public DossierTemplate importDossierTemplate(ImportTemplateResult request) {
public String importDossierTemplate(ImportTemplateResult request) {
String dossierTemplateId;
var dossierTemplateMeta = request.getDossierTemplate();
DossierTemplateEntity existingDossierTemplate = null;
@ -111,72 +109,42 @@ public class DossierTemplateImportService {
var watermarkNameToEntity = currentWatermarkConfigs.stream().collect(Collectors.toMap(WatermarkEntity::getName, Function.identity()));
request.getWatermarks().forEach(watermark -> {
try {
if (!watermarkNameToEntity.isEmpty() && watermarkNameToEntity.get(watermark.getName()) != null) {
watermark.setId(watermarkNameToEntity.get(watermark.getName()).getId());
}
watermarkService.createOrUpdateWatermark(watermark);
} catch (BadRequestException e) {
log.debug(" Conflict while validating watermark: " + watermark.getId());
if (!watermarkNameToEntity.isEmpty() && watermarkNameToEntity.get(watermark.getName()) != null) {
watermark.setId(watermarkNameToEntity.get(watermark.getName()).getId());
}
watermark.setDossierTemplateId(dossierTemplateId);
watermarkService.createOrUpdateWatermark(watermark);
});
configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId()));
}
// update the types
if (CollectionUtils.isNotEmpty(request.getTypes())) {
Set<String> toSetTypes = request.getTypes().stream().map(Type::getType).filter(Objects::nonNull).collect(Collectors.toSet());
List<TypeEntity> currentTypes = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, true);
Set<TypeEntity> typesToRemove = currentTypes.stream().filter(c -> !toSetTypes.contains(c.getType())).collect(Collectors.toSet());
var typeToEntityMap = currentTypes.stream().collect(Collectors.toMap(TypeEntity::getType, Function.identity()));
for (var type : request.getTypes()) {
type.setDossierTemplateId(dossierTemplateId);
String typeId;
if (!typeToEntityMap.isEmpty() && typeToEntityMap.get(type.getType()) != null) {
typeId = typeToEntityMap.get(type.getType()).getId();
dictionaryService.updateTypeValue(typeId, type);
} else {
var returnedType = dictionaryService.addType(type);
typeId = returnedType.getTypeId();
}
this.addEntries(request.getEntries(), typeId, type.getType(), DictionaryEntryType.ENTRY);
this.addEntries(request.getFalsePositives(), typeId, type.getType(), DictionaryEntryType.FALSE_POSITIVE);
this.addEntries(request.getFalseRecommendations(), typeId, type.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);
}
typesToRemove.forEach(type -> dictionaryService.deleteType(type.getId()));
this.updateTypes(request, dossierTemplateId);
} else { // no types to add, but remove existing ones
List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes();
currentTypes.forEach(type -> dictionaryService.deleteType(type.getId()));
}
// dossier status
if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
Set<String> toSetTypes = request.getDossierStatusInfos().stream().map(DossierStatusInfo::getName).filter(Objects::nonNull).collect(Collectors.toSet());
this.updateDossieStates(request, dossierTemplateId);
} else { // no states to add, delete current states
List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
Set<DossierStatusInfo> statesToRemove = currentStates.stream().filter(c -> !toSetTypes.contains(c.getName())).collect(Collectors.toSet());
var stateToEntityMap = currentStates.stream().collect(Collectors.toMap(DossierStatusInfo::getName, Function.identity()));
for (var state : request.getDossierStatusInfos()) {
state.setDossierTemplateId(dossierTemplateId);
if (!stateToEntityMap.isEmpty() && stateToEntityMap.get(state.getName()) != null) {
state.setId(stateToEntityMap.get(state.getName()).getId());
}
updateDossierStates(dossierTemplateId, state);
}
statesToRemove.forEach(state -> dossierStatusPersistenceService.deleteDossierStatus(state.getId(), null));
currentStates.forEach(state -> dossierStatusPersistenceService.deleteDossierStatus(state.getId(), null));
}
} else {
// creates new dossier template
if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
throw new ConflictException("DossierTemplate name must be set");
}
dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName());
DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
dossierTemplateEntity.setId(UUID.randomUUID().toString());
// order is important
BeanUtils.copyProperties(dossierTemplateMeta, dossierTemplateEntity);
dossierTemplateEntity.setId(UUID.randomUUID().toString());
dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
dossierTemplateEntity.setCreatedBy(request.getUserId());
//set rules
@ -192,11 +160,8 @@ public class DossierTemplateImportService {
if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
request.getWatermarks().forEach(watermark -> {
watermark.setDossierTemplateId(dossierTemplateId);
try {
watermarkService.createOrUpdateWatermark(watermark);
} catch (BadRequestException e) {
log.debug(" Conflict while validating watermark: " + watermark.getId());
}
watermark.setId(null);
watermarkService.createOrUpdateWatermark(watermark);
});
}
@ -204,21 +169,19 @@ public class DossierTemplateImportService {
if (CollectionUtils.isNotEmpty(request.getTypes())) {
for (var type : request.getTypes()) {
type.setDossierTemplateId(dossierTemplateId);
try {
var returnedType = dictionaryService.addType(type);
this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY);
this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE);
this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);
} catch (ConflictException e) {
log.debug("conflict while creating the types", e);
}
var returnedType = dictionaryService.addType(type);
this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY);
this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE);
this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);
}
}
// dossier status
if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
request.getDossierStatusInfos().forEach(state -> this.updateDossierStates(dossierTemplateId, state));
request.getDossierStatusInfos().forEach(state -> {
state.setId(null);
this.updateDossierStatus(dossierTemplateId, state);
});
}
}
// set legal basis
@ -248,12 +211,20 @@ public class DossierTemplateImportService {
reportTemplateService.uploadTemplate(reportRequest);
});
}
return convert(dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId), DossierTemplate.class);
return dossierTemplateId;
}
private void addEntries(Map<String, List<String>> entries, String typeId, String typeType, DictionaryEntryType dictionaryType) {
if (entries != null && !entries.isEmpty() && !entries.get(typeType).isEmpty()) {
dictionaryService.addEntries(typeId, entries.get(typeType), true, true, dictionaryType);
private void addEntries(Map<String, List<String>> entries, String typeId, String initialTypeId, DictionaryEntryType dictionaryType) {
if (entries != null && !entries.isEmpty() && entries.get(initialTypeId) != null && !entries.get(initialTypeId).isEmpty()) {
dictionaryService.addEntries(typeId, entries.get(initialTypeId), true, true, dictionaryType);
} else { // no entries, delete current entries
List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryType, null)
.stream()
.map(BaseDictionaryEntry::getValue)
.collect(Collectors.toList());
if (!existing.isEmpty()) {
dictionaryService.deleteEntries(typeId, existing, dictionaryType);
}
}
}
@ -283,24 +254,73 @@ public class DossierTemplateImportService {
}
private void updateDossierStates(String dossierTemplateId, DossierStatusInfo state) {
var dossierStatusRequest = CreateOrUpdateDossierStatusRequest.builder()
.dossierStatusId(state.getId())
.name(state.getName())
.description(state.getDescription())
.color(state.getColor())
.dossierTemplateId(dossierTemplateId)
.rank(state.getRank())
.build();
try {
dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
} catch (BadRequestException e) {
log.debug(" Empty name for dossier status id " + state.getId());
}catch (ConflictException e) {
log.debug(" Conflict detected for dossier status id: " + state.getId());
} catch (NotFoundException e) {
log.debug(" Not found dossier status id " + state.getId());
private DossierStatusEntity updateDossierStatus(String dossierTemplateId, DossierStatusInfo state) {
var dossierStatusRequest = CreateOrUpdateDossierStatusRequest.builder()
.dossierStatusId(state.getId())
.name(state.getName())
.description(state.getDescription())
.color(state.getColor())
.dossierTemplateId(dossierTemplateId)
.rank(state.getRank())
.build();
return dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
}
private void updateTypes(ImportTemplateResult request, String dossierTemplateId) {
List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes();
Set<String> currentTypesId = currentTypes.stream().map(TypeEntity::getId).collect(Collectors.toSet());
Set<String> typeIdsAdded = new HashSet<>();
var typeToEntityMap = currentTypes.stream().collect(Collectors.toMap(TypeEntity::getType, Function.identity()));
for (var type : request.getTypes()) {
// log.info("Adding type: " + type.getType() + " id: " + type.getId());
type.setDossierTemplateId(dossierTemplateId);
String typeId;
if (currentTypesId.contains(type.getId())) { //check by id
typeId = type.getId();
dictionaryService.updateTypeValue(typeId, type);
} else if (!typeToEntityMap.isEmpty() && typeToEntityMap.get(type.getType()) != null) { //check by name
typeId = typeToEntityMap.get(type.getType()).getId();
dictionaryService.updateTypeValue(typeId, type);
} else { // add the type, no match was found
var returnedType = dictionaryService.addType(type);
typeId = returnedType.getTypeId();
}
typeIdsAdded.add(typeId);
this.addEntries(request.getEntries(), typeId, type.getTypeId(), DictionaryEntryType.ENTRY);
this.addEntries(request.getFalsePositives(), typeId, type.getTypeId(), DictionaryEntryType.FALSE_POSITIVE);
this.addEntries(request.getFalseRecommendations(), typeId, type.getTypeId(), DictionaryEntryType.FALSE_RECOMMENDATION);
}
// remove additional types
Set<String> typesToRemove = currentTypesId.stream().filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet());
typesToRemove.forEach(dictionaryService::deleteType);
}
private void updateDossieStates(ImportTemplateResult request, String dossierTemplateId) {
List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
Set<String> currentStatesId = currentStates.stream().map(DossierStatusInfo::getId).collect(Collectors.toSet());
Set<String> stateIdsAdded = new HashSet<>();
var stateToEntityMap = currentStates.stream().collect(Collectors.toMap(DossierStatusInfo::getName, Function.identity()));
for (var state : request.getDossierStatusInfos()) {
state.setDossierTemplateId(dossierTemplateId);
String stateId;
if (currentStatesId.contains(state.getId())) { //check by id
stateId = state.getId();
} else if (!stateToEntityMap.isEmpty() && stateToEntityMap.get(state.getName()) != null) { //check by name
state.setId(stateToEntityMap.get(state.getName()).getId()); // force the id found to update the dossier status
} else { // add the state, no match was found
state.setId(null); // force null to create the dossier status
}
stateId = this.updateDossierStatus(dossierTemplateId, state).getId();
stateIdsAdded.add(stateId);
}
// remove additional states
Set<String> statesToRemove = currentStatesId.stream().filter(t -> !stateIdsAdded.contains(t)).collect(Collectors.toSet());
statesToRemove.forEach(stateId -> dossierStatusPersistenceService.deleteDossierStatus(stateId, null));
}
public ImportTemplateResult handleArchive(ImportDossierTemplateRequest request) {
@ -401,30 +421,37 @@ public class DossierTemplateImportService {
} else if (ze.getName().contains(ExportFilename.ENTRIES.getFilename())) {
List<String> entries = this.readEntries(bytes);
typeEntriesMap.put(this.getTypeName(ze.getName()), entries);
importTemplateResult.setEntries(typeEntriesMap);
typeEntriesMap.put(this.getTypeId(ze.getName()), entries);
} else if (ze.getName().contains(ExportFilename.FALSE_POSITIVES.getFilename())) {
List<String> falsePositives = this.readEntries(bytes);
typeFalsePositivesMap.put(this.getTypeName(ze.getName()), falsePositives);
importTemplateResult.setFalsePositives(typeFalsePositivesMap);
typeFalsePositivesMap.put(this.getTypeId(ze.getName()), falsePositives);
} else if (ze.getName().contains(ExportFilename.FALSE_RECOMMENDATION.getFilename())) {
List<String> falseRecommendations = this.readEntries(bytes);
typeFalseRecommendationsMap.put(this.getTypeName(ze.getName()), falseRecommendations);
importTemplateResult.setFalseRecommendations(typeFalseRecommendationsMap);
} else if (ze.getName().contains("reportTemplateList.json")) {
typeFalseRecommendationsMap.put(this.getTypeId(ze.getName()), falseRecommendations);
} else if (ze.getName().contains(ExportFilename.REPORT_TEMPLATE.getFilename())) {
var reportTemplateList = objectMapper.readValue(bytes, new TypeReference<List<ReportTemplate>>() {
});
reportTemplateMap = reportTemplateList.stream()
.collect(Collectors.toMap(ReportTemplate::getFileName, Function.identity()));
reportTemplateFilenameList = reportTemplateList.stream().map(ReportTemplate::getFileName).collect(Collectors.toList());
.collect(Collectors.toMap(rt ->
rt.isMultiFileReport()? rt.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : rt.getFileName(),
Function.identity()));
reportTemplateFilenameList = reportTemplateList.stream().map(rt ->
rt.isMultiFileReport()? rt.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : rt.getFileName()).collect(Collectors.toList());
} else {
reportTemplateBytesMap.put(ze.getName(), bos);
reportTemplateBytesMap.put(ze.getName(), bos);
}
bos.close();
}
}
importTemplateResult.setEntries(typeEntriesMap);
importTemplateResult.setFalsePositives(typeFalsePositivesMap);
importTemplateResult.setFalseRecommendations(typeFalseRecommendationsMap);
List<ReportTemplateUploadRequest> reportTemplateUploadRequests = new ArrayList<>();
for (var reportZe : reportTemplateBytesMap.entrySet()) {
@ -469,11 +496,12 @@ public class DossierTemplateImportService {
}
} catch (IOException e) {
log.debug("exception: ", e);
throw new BadRequestException("Error while reading the entries", e);
}
return entries;
}
private String getTypeName(String filename) {
private String getTypeId(String filename) {
var index = filename.indexOf('/');
return filename.substring(0, index);
}

View File

@ -32,6 +32,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.io.*;
import java.util.List;
import java.util.stream.Collectors;
@ -72,6 +73,7 @@ public class DossierTemplateExportService {
}
@SneakyThrows
@Transactional
public void createDownloadArchive(DownloadJob downloadJob) {
objectMapper.registerModule(new JavaTimeModule());
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId());
@ -130,14 +132,17 @@ public class DossierTemplateExportService {
objectMapper.writeValueAsBytes(convert(reportTemplateList, ReportTemplate.class))));
reportTemplateList.forEach(reportTemplate -> {
var report = fileManagementStorageService.getStoredObjectBytes(reportTemplate.getStorageId());
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, reportTemplate.getFileName(), report));
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null,
reportTemplate.isMultiFileReport()? reportTemplate.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : reportTemplate.getFileName(),
report));
});
// for every type 1 folder with:
// 1 json file containing meta info of the type
// and 1 txt file for every type: entries, false positives and false recommendation
var dossierTypes = dossierTemplate.getDossierTypes();
for (TypeEntity typeEntity : dossierTypes) {
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getType(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT),
// log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId());
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getId(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT),
objectMapper.writeValueAsBytes(convert(typeEntity, Type.class))));
var entriesValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.ENTRY, null)
@ -195,7 +200,7 @@ public class DossierTemplateExportService {
private void addToExportDownloadQueue(DownloadJob downloadJob, int priority) {
try {
rabbitTemplate.convertAndSend(MessagingConfiguration.DOWNLOAD_QUEUE, objectMapper.writeValueAsString(downloadJob), message -> {
rabbitTemplate.convertAndSend(MessagingConfiguration.EXPORT_DOWNLOAD_QUEUE, objectMapper.writeValueAsString(downloadJob), message -> {
message.getMessageProperties().setPriority(priority);
return message;
});

View File

@ -411,7 +411,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
var newDossierTemplate = dossierTemplateClient.importDossierTemplate(request1);
var allDossierTemplates = dossierTemplateClient.getAllDossierTemplates();
assertThat(allDossierTemplates.size()).isEqualTo(1); //should be 2
assertThat(allDossierTemplates.size()).isEqualTo(2);
}
@ -450,7 +450,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
var newDossierTemplate = dossierTemplateClient.importDossierTemplate(request1);
var allDossierTemplates = dossierTemplateClient.getAllDossierTemplates();
assertThat(allDossierTemplates.size()).isEqualTo(1); // should be 2
assertThat(allDossierTemplates.size()).isEqualTo(2);
}