From 870673f679ceaad22150eac0c00a43009c127668 Mon Sep 17 00:00:00 2001 From: devplant Date: Wed, 28 Sep 2022 15:15:42 +0300 Subject: [PATCH] RED-2200 - Dossier Template export and import - rework the update of types. Include in export the types not dossier related and the ones that are not deleted. - at import and existing template for existing types which are not included in archive that are system managed just delete the entries - in case of restoring a deleted type, undelete it first and then update the type - when import as a new template and template's name is not valid, set a valid one - download cleanup job updated to fix null pointer exception for export downloads --- .../DictionaryPersistenceService.java | 11 +- .../repository/TypeRepository.java | 4 + .../v1/server/service/DictionaryService.java | 14 ++- .../service/DossierTemplateImportService.java | 100 +++++++++++++----- .../export/DossierTemplateExportService.java | 14 ++- .../service/job/DownloadCleanupJob.java | 2 +- 6 files changed, 106 insertions(+), 39 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java index a103f5188..f8aa0863e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java @@ -72,9 +72,9 @@ public class DictionaryPersistenceService { public void updateType(String typeId, TypeEntity typeValueRequest) { typeRepository.findById(typeId).ifPresent(type -> { - if (type.isDeleted()) { - throw new NotFoundException("Type is deleted!"); - } +// if (type.isDeleted()) { +// throw new NotFoundException("Type is deleted!"); +// } type.setVersion(type.getVersion() + 1); checkRankAlreadyExists(type.getType(), type.getDossierTemplate().getId(), typeValueRequest.getRank(), type.getDossier() == null ? null : type.getDossier().getId()); @@ -223,4 +223,9 @@ public class DictionaryPersistenceService { typeRepository.saveAll(types); } + @Transactional + public int undeleteType(String typeId) { + return typeRepository.unSoftDeleteTypeById(typeId); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java index 847656b24..fe8d27a50 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java @@ -47,4 +47,8 @@ public interface TypeRepository extends JpaRepository { @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("Update TypeEntity t set t.softDeletedTime = CURRENT_TIMESTAMP where t.id = :typeId") void softDeleteTypeById(String typeId); + + @Modifying + @Query("Update TypeEntity t set t.softDeletedTime = null where t.id = :typeId and t.softDeletedTime is not null") + int unSoftDeleteTypeById(String typeId); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java index 7a0e12a99..646dce8b4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java @@ -6,6 +6,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; @@ -98,7 +99,7 @@ public class DictionaryService { } - // To check whether the type exists + // To check whether the type exists and it is not deleted Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); if (typeRequest.getLabel() != null) { @@ -197,6 +198,17 @@ public class DictionaryService { dictionaryPersistenceService.incrementVersion(typeId); } + public long getCurrentVersion(String typeId) { + Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); + long currentVersion; + if (typeResult.getDossierId() != null) { + currentVersion = dictionaryPersistenceService.getVersionForDossier(typeResult.getDossierId()); + } else { + currentVersion = dictionaryPersistenceService.getVersion(typeResult.getDossierTemplateId()); + } + return currentVersion; + } + private void validateBoolean(Boolean bool, String name) { Optional errorMessage = DictionaryValidator.validateBoolean(bool, name); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java index a0fa467e6..d3e8a6ab1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java @@ -118,14 +118,6 @@ public class DossierTemplateImportService { configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId())); } - // update the types - if (CollectionUtils.isNotEmpty(request.getTypes())) { - this.updateTypes(request, dossierTemplateId); - } else { // no types to add, but remove existing ones - List currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes(); - currentTypes.forEach(type -> dictionaryService.deleteType(type.getId())); - } - // dossier status if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) { this.updateDossierStates(request, dossierTemplateId); @@ -150,12 +142,21 @@ public class DossierTemplateImportService { currentConfigs.forEach(fa -> fileAttributeConfigPersistenceService.deleteFileAttribute(fa.getId())); } + // update the types + if (CollectionUtils.isNotEmpty(request.getTypes())) { + this.updateTypes(request, dossierTemplateId); + } else { // no types to add, but remove existing ones + List currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes() + .stream().filter(t -> t.getDossierId() == null).collect(Collectors.toList()); + this.deleteTypes(currentTypes, new HashSet<>()); + } + } else { // creates new dossier template if (StringUtils.isEmpty(dossierTemplateMeta.getName())) { throw new ConflictException("DossierTemplate name must be set"); } - dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName()); + this.validateDossierTemplateName(dossierTemplateMeta); DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity(); // order is important @@ -181,17 +182,7 @@ public class DossierTemplateImportService { }); } - //set types - if (CollectionUtils.isNotEmpty(request.getTypes())) { - for (var type : request.getTypes()) { - type.setDossierTemplateId(dossierTemplateId); - var returnedType = dictionaryService.addType(type); - this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY); - this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE); - this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION); - } - } // dossier status if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) { request.getDossierStatusInfos().forEach(state -> { @@ -215,6 +206,18 @@ public class DossierTemplateImportService { fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId, convert(fa, FileAttributeConfigEntity.class)); }); } + + //set types + if (CollectionUtils.isNotEmpty(request.getTypes())) { + for (var type : request.getTypes()) { + type.setDossierTemplateId(dossierTemplateId); + var returnedType = dictionaryService.addType(type); + this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY); + this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE); + this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION); + + } + } } // set legal basis if (CollectionUtils.isNotEmpty(request.getLegalBases())) { @@ -289,8 +292,28 @@ public class DossierTemplateImportService { return dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest); } + private void validateDossierTemplateName(DossierTemplate dossierTemplateMeta) { + boolean cond = true; + int index = 0; + String dossierTemplateName = dossierTemplateMeta.getName(); + do { + try { + dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName()); + cond = false; + } catch (ConflictException e) { + if (index == 0) { + dossierTemplateMeta.setName("Copy of " + dossierTemplateName); + } else { + dossierTemplateMeta.setName("Copy of " + dossierTemplateName + " - " + index); + } + index++; + } + } while (cond); + } + private void updateTypes(ImportTemplateResult request, String dossierTemplateId) { - List currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes(); + List currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes() + .stream().filter(t -> t.getDossierId() == null).collect(Collectors.toList()); Set currentTypesId = currentTypes.stream().map(TypeEntity::getId).collect(Collectors.toSet()); Set typeIdsAdded = new HashSet<>(); @@ -301,9 +324,13 @@ public class DossierTemplateImportService { String typeId; if (currentTypesId.contains(type.getId())) { //check by id typeId = type.getId(); + // type found, possible to be deleted, undelete it first before updating + dictionaryPersistenceService.undeleteType(typeId); dictionaryService.updateTypeValue(typeId, type); } else if (!typeToEntityMap.isEmpty() && typeToEntityMap.get(type.getType()) != null) { //check by name typeId = typeToEntityMap.get(type.getType()).getId(); + // type found, possible to be deleted, undelete it first before updating + dictionaryPersistenceService.undeleteType(typeId); dictionaryService.updateTypeValue(typeId, type); } else { // add the type, no match was found var returnedType = dictionaryService.addType(type); @@ -315,11 +342,26 @@ public class DossierTemplateImportService { this.addEntries(request.getFalsePositives(), typeId, type.getTypeId(), DictionaryEntryType.FALSE_POSITIVE); this.addEntries(request.getFalseRecommendations(), typeId, type.getTypeId(), DictionaryEntryType.FALSE_RECOMMENDATION); } - // remove additional types - Set typesToRemove = currentTypesId.stream().filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet()); - typesToRemove.forEach(dictionaryService::deleteType); + // remove additional types and not included in the archive + this.deleteTypes(currentTypes, typeIdsAdded); } + private void deleteTypes(List currentTypes, Set typeIdsAdded) { + Set currentTypesIdSystemManaged = currentTypes.stream().filter(TypeEntity::isSystemManaged).map(TypeEntity::getId).collect(Collectors.toSet()); + + // for types system managed just delete the entries + Set entriesToRemove = currentTypesIdSystemManaged.stream().filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet()); + entriesToRemove.forEach(typeId -> { + var currentVersion = dictionaryService.getCurrentVersion(typeId); + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY); + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); + dictionaryPersistenceService.incrementVersion(typeId); + typeIdsAdded.add(typeId); // added to the list, since the type can not be deleted + }); + Set typesToRemove = currentTypes.stream().map(TypeEntity::getId).filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet()); + typesToRemove.forEach(dictionaryService::deleteType); + } private void updateDossierStates(ImportTemplateResult request, String dossierTemplateId) { List currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId); @@ -358,7 +400,7 @@ public class DossierTemplateImportService { }); // remove existing dossier attributes and not included in archive Set attributesToRemove = currentDossierAttributeIds.stream().filter(da -> !dossierAttributeIdsAdded.contains(da)).collect(Collectors.toSet()); - attributesToRemove.forEach(daId -> dossierAttributeConfigPersistenceService.deleteDossierAttribute(daId)); + attributesToRemove.forEach(dossierAttributeConfigPersistenceService::deleteDossierAttribute); } private void updateFileAttributes(ImportTemplateResult request, String dossierTemplateId) { @@ -373,7 +415,7 @@ public class DossierTemplateImportService { }); // remove existing file attributes and not included in archive Set attributesToRemove = currentFileAttributeIds.stream().filter(fa -> !fileAttributeIdsAdded.contains(fa)).collect(Collectors.toSet()); - attributesToRemove.forEach(faId -> fileAttributeConfigPersistenceService.deleteFileAttribute(faId)); + attributesToRemove.forEach(fileAttributeConfigPersistenceService::deleteFileAttribute); } public ImportTemplateResult handleArchive(ImportDossierTemplateRequest request) { @@ -474,17 +516,17 @@ public class DossierTemplateImportService { } else if (ze.getName().contains(ExportFilename.ENTRIES.getFilename())) { List entries = this.readEntries(bytes); - typeEntriesMap.put(this.getTypeId(ze.getName()), entries); + typeEntriesMap.put(this.getType(ze.getName()), entries); } else if (ze.getName().contains(ExportFilename.FALSE_POSITIVES.getFilename())) { List falsePositives = this.readEntries(bytes); - typeFalsePositivesMap.put(this.getTypeId(ze.getName()), falsePositives); + typeFalsePositivesMap.put(this.getType(ze.getName()), falsePositives); } else if (ze.getName().contains(ExportFilename.FALSE_RECOMMENDATION.getFilename())) { List falseRecommendations = this.readEntries(bytes); - typeFalseRecommendationsMap.put(this.getTypeId(ze.getName()), falseRecommendations); + typeFalseRecommendationsMap.put(this.getType(ze.getName()), falseRecommendations); } else if (ze.getName().contains(ExportFilename.REPORT_TEMPLATE.getFilename())) { var reportTemplateList = objectMapper.readValue(bytes, new TypeReference>() { @@ -554,7 +596,7 @@ public class DossierTemplateImportService { return entries; } - private String getTypeId(String filename) { + private String getType(String filename) { var index = filename.indexOf('/'); return filename.substring(0, index); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java index 20cc5cac8..e4c019fae 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java @@ -139,10 +139,14 @@ public class DossierTemplateExportService { // for every type 1 folder with: // 1 json file containing meta info of the type // and 1 txt file for every type: entries, false positives and false recommendation - var dossierTypes = dossierTemplate.getDossierTypes(); + + // remove the types related to dossiers and also the ones that are deleted + var dossierTypes = dossierTemplate.getDossierTypes().stream() + .filter(t -> t.getDossierId() == null) + .filter(t -> !t.isDeleted()).collect(Collectors.toList()); for (TypeEntity typeEntity : dossierTypes) { -// log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId()); - fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getId(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT), +// log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId() + " " + typeEntity.isDeleted()); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getType(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT), objectMapper.writeValueAsBytes(convert(typeEntity, Type.class)))); var entriesValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.ENTRY, null) @@ -153,8 +157,8 @@ public class DossierTemplateExportService { .stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList()); writeEntriesListToFile(fileSystemBackedArchiver, entriesValuesList, typeEntity.getType(), getFilename(ExportFilename.ENTRIES, TXT_EXT)); - writeEntriesListToFile(fileSystemBackedArchiver, falsePositiveValuesList, typeEntity.getType(),getFilename(ExportFilename.FALSE_POSITIVES, TXT_EXT)); - writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(),getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT)); + writeEntriesListToFile(fileSystemBackedArchiver, falsePositiveValuesList, typeEntity.getType(), getFilename(ExportFilename.FALSE_POSITIVES, TXT_EXT)); + writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(), getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT)); } storeZipFile(downloadStatus.getStorageId(), fileSystemBackedArchiver); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/DownloadCleanupJob.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/DownloadCleanupJob.java index 77b5c6fea..4958eafcd 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/DownloadCleanupJob.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/DownloadCleanupJob.java @@ -56,7 +56,7 @@ public class DownloadCleanupJob implements Job { log.info("2. Deleting download status {} because DownloadCleanupNotDownloadFilesHours is {} and c+h {} is after {}", downloadStatus, applicationConfigurationEntity.getDownloadCleanupNotDownloadFilesHours(), downloadStatus.getCreationDate() .plusHours(applicationConfigurationEntity.getDownloadCleanupNotDownloadFilesHours()), now); deleteDownload(downloadStatus); - } else if (dossierService.getDossierById(dossier.getId()).getSoftDeletedTime() != null) { + } else if (dossier != null && dossierService.getDossierById(dossier.getId()).getSoftDeletedTime() != null) { log.info("3. Deleting download {}, because dossier does not exist", downloadStatus.getStorageId()); deleteDownload(downloadStatus); }