Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #540: RED-2200 - Dossier Template export and import

Browse Source 
Merge in RED/persistence-service from RED-2200_5 to master

* commit '870673f679ceaad22150eac0c00a43009c127668':
  RED-2200 - Dossier Template export and import
This commit is contained in:
Corina Olariu 2022-09-28 16:43:48 +02:00 committed by Timo Bejan
commit 618b5dc0bb
6 changed files with 106 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java
View File

@ -72,9 +72,9 @@ public class DictionaryPersistenceService {
public void updateType(String typeId, TypeEntity typeValueRequest) {
typeRepository.findById(typeId).ifPresent(type -> {
if (type.isDeleted()) {
throw new NotFoundException("Type is deleted!");
}
// if (type.isDeleted()) {
// throw new NotFoundException("Type is deleted!");
// }
type.setVersion(type.getVersion() + 1);
checkRankAlreadyExists(type.getType(), type.getDossierTemplate().getId(), typeValueRequest.getRank(), type.getDossier() == null ? null : type.getDossier().getId());
@ -223,4 +223,9 @@ public class DictionaryPersistenceService {
typeRepository.saveAll(types);
}
@Transactional
public int undeleteType(String typeId) {
return typeRepository.unSoftDeleteTypeById(typeId);
}
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java
View File

@ -48,4 +48,8 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
@Modifying(clearAutomatically = true, flushAutomatically = true)
@Query("Update TypeEntity t set t.softDeletedTime = CURRENT_TIMESTAMP where t.id = :typeId")
void softDeleteTypeById(String typeId);
@Modifying
@Query("Update TypeEntity t set t.softDeletedTime = null where t.id = :typeId and t.softDeletedTime is not null")
int unSoftDeleteTypeById(String typeId);
}

14
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java
View File

@ -21,6 +21,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
@ -100,7 +101,7 @@ public class DictionaryService {
}
// To check whether the type exists
// To check whether the type exists and it is not deleted
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
if (typeRequest.getLabel() != null) {
@ -199,6 +200,17 @@ public class DictionaryService {
dictionaryPersistenceService.incrementVersion(typeId);
}
public long getCurrentVersion(String typeId) {
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
long currentVersion;
if (typeResult.getDossierId() != null) {
currentVersion = dictionaryPersistenceService.getVersionForDossier(typeResult.getDossierId());
} else {
currentVersion = dictionaryPersistenceService.getVersion(typeResult.getDossierTemplateId());
}
return currentVersion;
}
private void validateBoolean(Boolean bool, String name) {
Optional<String> errorMessage = DictionaryValidator.validateBoolean(bool, name);

100
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java
View File

@ -118,14 +118,6 @@ public class DossierTemplateImportService {
configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId()));
}
// update the types
if (CollectionUtils.isNotEmpty(request.getTypes())) {
this.updateTypes(request, dossierTemplateId);
} else { // no types to add, but remove existing ones
List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes();
currentTypes.forEach(type -> dictionaryService.deleteType(type.getId()));
}
// dossier status
if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
this.updateDossierStates(request, dossierTemplateId);
@ -150,12 +142,21 @@ public class DossierTemplateImportService {
currentConfigs.forEach(fa -> fileAttributeConfigPersistenceService.deleteFileAttribute(fa.getId()));
}
// update the types
if (CollectionUtils.isNotEmpty(request.getTypes())) {
this.updateTypes(request, dossierTemplateId);
} else { // no types to add, but remove existing ones
List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes()
.stream().filter(t -> t.getDossierId() == null).collect(Collectors.toList());
this.deleteTypes(currentTypes, new HashSet<>());
}
} else {
// creates new dossier template
if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
throw new ConflictException("DossierTemplate name must be set");
}
dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName());
this.validateDossierTemplateName(dossierTemplateMeta);
DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
// order is important
@ -181,17 +182,7 @@ public class DossierTemplateImportService {
});
}
//set types
if (CollectionUtils.isNotEmpty(request.getTypes())) {
for (var type : request.getTypes()) {
type.setDossierTemplateId(dossierTemplateId);
var returnedType = dictionaryService.addType(type);
this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY);
this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE);
this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);
}
}
// dossier status
if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
request.getDossierStatusInfos().forEach(state -> {
@ -215,6 +206,18 @@ public class DossierTemplateImportService {
fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId, convert(fa, FileAttributeConfigEntity.class));
});
}
//set types
if (CollectionUtils.isNotEmpty(request.getTypes())) {
for (var type : request.getTypes()) {
type.setDossierTemplateId(dossierTemplateId);
var returnedType = dictionaryService.addType(type);
this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY);
this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE);
this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);
}
}
}
// set legal basis
if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
@ -289,8 +292,28 @@ public class DossierTemplateImportService {
return dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
}
private void validateDossierTemplateName(DossierTemplate dossierTemplateMeta) {
boolean cond = true;
int index = 0;
String dossierTemplateName = dossierTemplateMeta.getName();
do {
try {
dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName());
cond = false;
} catch (ConflictException e) {
if (index == 0) {
dossierTemplateMeta.setName("Copy of " + dossierTemplateName);
} else {
dossierTemplateMeta.setName("Copy of " + dossierTemplateName + " - " + index);
}
index++;
}
} while (cond);
}
private void updateTypes(ImportTemplateResult request, String dossierTemplateId) {
List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes();
List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes()
.stream().filter(t -> t.getDossierId() == null).collect(Collectors.toList());
Set<String> currentTypesId = currentTypes.stream().map(TypeEntity::getId).collect(Collectors.toSet());
Set<String> typeIdsAdded = new HashSet<>();
@ -301,9 +324,13 @@ public class DossierTemplateImportService {
String typeId;
if (currentTypesId.contains(type.getId())) { //check by id
typeId = type.getId();
// type found, possible to be deleted, undelete it first before updating
dictionaryPersistenceService.undeleteType(typeId);
dictionaryService.updateTypeValue(typeId, type);
} else if (!typeToEntityMap.isEmpty() && typeToEntityMap.get(type.getType()) != null) { //check by name
typeId = typeToEntityMap.get(type.getType()).getId();
// type found, possible to be deleted, undelete it first before updating
dictionaryPersistenceService.undeleteType(typeId);
dictionaryService.updateTypeValue(typeId, type);
} else { // add the type, no match was found
var returnedType = dictionaryService.addType(type);
@ -315,11 +342,26 @@ public class DossierTemplateImportService {
this.addEntries(request.getFalsePositives(), typeId, type.getTypeId(), DictionaryEntryType.FALSE_POSITIVE);
this.addEntries(request.getFalseRecommendations(), typeId, type.getTypeId(), DictionaryEntryType.FALSE_RECOMMENDATION);
}
// remove additional types
Set<String> typesToRemove = currentTypesId.stream().filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet());
typesToRemove.forEach(dictionaryService::deleteType);
// remove additional types and not included in the archive
this.deleteTypes(currentTypes, typeIdsAdded);
}
private void deleteTypes(List<TypeEntity> currentTypes, Set<String> typeIdsAdded) {
Set<String> currentTypesIdSystemManaged = currentTypes.stream().filter(TypeEntity::isSystemManaged).map(TypeEntity::getId).collect(Collectors.toSet());
// for types system managed just delete the entries
Set<String> entriesToRemove = currentTypesIdSystemManaged.stream().filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet());
entriesToRemove.forEach(typeId -> {
var currentVersion = dictionaryService.getCurrentVersion(typeId);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
dictionaryPersistenceService.incrementVersion(typeId);
typeIdsAdded.add(typeId); // added to the list, since the type can not be deleted
});
Set<String> typesToRemove = currentTypes.stream().map(TypeEntity::getId).filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet());
typesToRemove.forEach(dictionaryService::deleteType);
}
private void updateDossierStates(ImportTemplateResult request, String dossierTemplateId) {
List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
@ -358,7 +400,7 @@ public class DossierTemplateImportService {
});
// remove existing dossier attributes and not included in archive
Set<String> attributesToRemove = currentDossierAttributeIds.stream().filter(da -> !dossierAttributeIdsAdded.contains(da)).collect(Collectors.toSet());
attributesToRemove.forEach(daId -> dossierAttributeConfigPersistenceService.deleteDossierAttribute(daId));
attributesToRemove.forEach(dossierAttributeConfigPersistenceService::deleteDossierAttribute);
}
private void updateFileAttributes(ImportTemplateResult request, String dossierTemplateId) {
@ -373,7 +415,7 @@ public class DossierTemplateImportService {
});
// remove existing file attributes and not included in archive
Set<String> attributesToRemove = currentFileAttributeIds.stream().filter(fa -> !fileAttributeIdsAdded.contains(fa)).collect(Collectors.toSet());
attributesToRemove.forEach(faId -> fileAttributeConfigPersistenceService.deleteFileAttribute(faId));
attributesToRemove.forEach(fileAttributeConfigPersistenceService::deleteFileAttribute);
}
public ImportTemplateResult handleArchive(ImportDossierTemplateRequest request) {
@ -474,17 +516,17 @@ public class DossierTemplateImportService {
} else if (ze.getName().contains(ExportFilename.ENTRIES.getFilename())) {
List<String> entries = this.readEntries(bytes);
typeEntriesMap.put(this.getTypeId(ze.getName()), entries);
typeEntriesMap.put(this.getType(ze.getName()), entries);
} else if (ze.getName().contains(ExportFilename.FALSE_POSITIVES.getFilename())) {
List<String> falsePositives = this.readEntries(bytes);
typeFalsePositivesMap.put(this.getTypeId(ze.getName()), falsePositives);
typeFalsePositivesMap.put(this.getType(ze.getName()), falsePositives);
} else if (ze.getName().contains(ExportFilename.FALSE_RECOMMENDATION.getFilename())) {
List<String> falseRecommendations = this.readEntries(bytes);
typeFalseRecommendationsMap.put(this.getTypeId(ze.getName()), falseRecommendations);
typeFalseRecommendationsMap.put(this.getType(ze.getName()), falseRecommendations);
} else if (ze.getName().contains(ExportFilename.REPORT_TEMPLATE.getFilename())) {
var reportTemplateList = objectMapper.readValue(bytes, new TypeReference<List<ReportTemplate>>() {
@ -554,7 +596,7 @@ public class DossierTemplateImportService {
return entries;
}
private String getTypeId(String filename) {
private String getType(String filename) {
var index = filename.indexOf('/');
return filename.substring(0, index);
}

14
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java
View File

@ -139,10 +139,14 @@ public class DossierTemplateExportService {
// for every type 1 folder with:
// 1 json file containing meta info of the type
// and 1 txt file for every type: entries, false positives and false recommendation
var dossierTypes = dossierTemplate.getDossierTypes();
// remove the types related to dossiers and also the ones that are deleted
var dossierTypes = dossierTemplate.getDossierTypes().stream()
.filter(t -> t.getDossierId() == null)
.filter(t -> !t.isDeleted()).collect(Collectors.toList());
for (TypeEntity typeEntity : dossierTypes) {
// log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId());
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getId(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT),
// log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId() + " " + typeEntity.isDeleted());
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getType(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT),
objectMapper.writeValueAsBytes(convert(typeEntity, Type.class))));
var entriesValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.ENTRY, null)
@ -153,8 +157,8 @@ public class DossierTemplateExportService {
.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList());
writeEntriesListToFile(fileSystemBackedArchiver, entriesValuesList, typeEntity.getType(), getFilename(ExportFilename.ENTRIES, TXT_EXT));
writeEntriesListToFile(fileSystemBackedArchiver, falsePositiveValuesList, typeEntity.getType(),getFilename(ExportFilename.FALSE_POSITIVES, TXT_EXT));
writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(),getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT));
writeEntriesListToFile(fileSystemBackedArchiver, falsePositiveValuesList, typeEntity.getType(), getFilename(ExportFilename.FALSE_POSITIVES, TXT_EXT));
writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(), getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT));
}
storeZipFile(downloadStatus.getStorageId(), fileSystemBackedArchiver);

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/DownloadCleanupJob.java
View File

@ -56,7 +56,7 @@ public class DownloadCleanupJob implements Job {
log.info("2. Deleting download status {} because DownloadCleanupNotDownloadFilesHours is {} and c+h {} is after {}", downloadStatus, applicationConfigurationEntity.getDownloadCleanupNotDownloadFilesHours(), downloadStatus.getCreationDate()
.plusHours(applicationConfigurationEntity.getDownloadCleanupNotDownloadFilesHours()), now);
deleteDownload(downloadStatus);
} else if (dossierService.getDossierById(dossier.getId()).getSoftDeletedTime() != null) {
} else if (dossier != null && dossierService.getDossierById(dossier.getId()).getSoftDeletedTime() != null) {
log.info("3. Deleting download {}, because dossier does not exist", downloadStatus.getStorageId());
deleteDownload(downloadStatus);
}