Pull request #540: RED-2200 - Dossier Template export and import

Merge in RED/persistence-service from RED-2200_5 to master

* commit '870673f679ceaad22150eac0c00a43009c127668':
  RED-2200 - Dossier Template export and import

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java

@@ -72,9 +72,9 @@ public class DictionaryPersistenceService {
     public void updateType(String typeId, TypeEntity typeValueRequest) {
 
         typeRepository.findById(typeId).ifPresent(type -> {
-            if (type.isDeleted()) {
+//            if (type.isDeleted()) {
-                throw new NotFoundException("Type is deleted!");
+//                throw new NotFoundException("Type is deleted!");
-            }
+//            }
 
             type.setVersion(type.getVersion() + 1);
             checkRankAlreadyExists(type.getType(), type.getDossierTemplate().getId(), typeValueRequest.getRank(), type.getDossier() == null ? null : type.getDossier().getId());
@@ -223,4 +223,9 @@ public class DictionaryPersistenceService {
         typeRepository.saveAll(types);
     }
 
+    @Transactional
+    public int undeleteType(String typeId) {
+        return typeRepository.unSoftDeleteTypeById(typeId);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java

@@ -48,4 +48,8 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
     @Modifying(clearAutomatically = true, flushAutomatically = true)
     @Query("Update TypeEntity t set t.softDeletedTime = CURRENT_TIMESTAMP where t.id = :typeId")
     void softDeleteTypeById(String typeId);
+
+    @Modifying
+    @Query("Update TypeEntity t set t.softDeletedTime = null where t.id = :typeId and t.softDeletedTime is not null")
+    int unSoftDeleteTypeById(String typeId);
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java

@@ -21,6 +21,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
@@ -100,7 +101,7 @@ public class DictionaryService {
         }
 
 
-        // To check whether the type exists
+        // To check whether the type exists and it is not deleted
         Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
 
         if (typeRequest.getLabel() != null) {
@@ -199,6 +200,17 @@ public class DictionaryService {
         dictionaryPersistenceService.incrementVersion(typeId);
     }
 
+    public long getCurrentVersion(String typeId) {
+        Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
+        long currentVersion;
+        if (typeResult.getDossierId() != null) {
+            currentVersion = dictionaryPersistenceService.getVersionForDossier(typeResult.getDossierId());
+        } else {
+            currentVersion = dictionaryPersistenceService.getVersion(typeResult.getDossierTemplateId());
+        }
+        return currentVersion;
+    }
+
     private void validateBoolean(Boolean bool, String name) {
 
         Optional<String> errorMessage = DictionaryValidator.validateBoolean(bool, name);

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java

@@ -118,14 +118,6 @@ public class DossierTemplateImportService {
                 configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId()));
             }
 
-            // update the types
-            if (CollectionUtils.isNotEmpty(request.getTypes())) {
-                this.updateTypes(request, dossierTemplateId);
-            } else { // no types to add, but remove existing ones
-                List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes();
-                currentTypes.forEach(type -> dictionaryService.deleteType(type.getId()));
-            }
-
             // dossier status
             if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
                 this.updateDossierStates(request, dossierTemplateId);
@@ -150,12 +142,21 @@ public class DossierTemplateImportService {
                 currentConfigs.forEach(fa -> fileAttributeConfigPersistenceService.deleteFileAttribute(fa.getId()));
             }
 
+            // update the types
+            if (CollectionUtils.isNotEmpty(request.getTypes())) {
+                this.updateTypes(request, dossierTemplateId);
+            } else { // no types to add, but remove existing ones
+                List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes()
+                        .stream().filter(t -> t.getDossierId() == null).collect(Collectors.toList());
+                this.deleteTypes(currentTypes, new HashSet<>());
+            }
+
         } else {
             // creates new dossier template
             if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
                 throw new ConflictException("DossierTemplate name must be set");
             }
-            dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName());
+            this.validateDossierTemplateName(dossierTemplateMeta);
             DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
 
             // order is important
@@ -181,17 +182,7 @@ public class DossierTemplateImportService {
                 });
             }
 
-            //set types
-            if (CollectionUtils.isNotEmpty(request.getTypes())) {
-                for (var type : request.getTypes()) {
-                    type.setDossierTemplateId(dossierTemplateId);
-                    var returnedType = dictionaryService.addType(type);
-                    this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY);
-                    this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE);
-                    this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);
 
-                }
-            }
             // dossier status
             if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
                 request.getDossierStatusInfos().forEach(state -> {
@@ -215,6 +206,18 @@ public class DossierTemplateImportService {
                     fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId, convert(fa, FileAttributeConfigEntity.class));
                 });
             }
+
+            //set types
+            if (CollectionUtils.isNotEmpty(request.getTypes())) {
+                for (var type : request.getTypes()) {
+                    type.setDossierTemplateId(dossierTemplateId);
+                    var returnedType = dictionaryService.addType(type);
+                    this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY);
+                    this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE);
+                    this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);
+
+                }
+            }
         }
         // set legal basis
         if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
@@ -289,8 +292,28 @@ public class DossierTemplateImportService {
             return dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
     }
 
+    private void validateDossierTemplateName(DossierTemplate dossierTemplateMeta) {
+        boolean cond = true;
+        int index = 0;
+        String dossierTemplateName = dossierTemplateMeta.getName();
+        do {
+            try {
+                dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName());
+                cond = false;
+            } catch (ConflictException e) {
+                if (index == 0) {
+                    dossierTemplateMeta.setName("Copy of " + dossierTemplateName);
+                } else {
+                    dossierTemplateMeta.setName("Copy of " + dossierTemplateName + " - " + index);
+                }
+                index++;
+            }
+        } while (cond);
+    }
+
     private void updateTypes(ImportTemplateResult request, String dossierTemplateId) {
-        List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes();
+        List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getDossierTypes()
+                .stream().filter(t -> t.getDossierId() == null).collect(Collectors.toList());
 
         Set<String> currentTypesId = currentTypes.stream().map(TypeEntity::getId).collect(Collectors.toSet());
         Set<String> typeIdsAdded = new HashSet<>();
@@ -301,9 +324,13 @@ public class DossierTemplateImportService {
             String typeId;
             if (currentTypesId.contains(type.getId())) { //check by id
                 typeId = type.getId();
+                // type found, possible to be deleted, undelete it first before updating
+                dictionaryPersistenceService.undeleteType(typeId);
                 dictionaryService.updateTypeValue(typeId, type);
             } else if (!typeToEntityMap.isEmpty() && typeToEntityMap.get(type.getType()) != null) { //check by name
                 typeId = typeToEntityMap.get(type.getType()).getId();
+                // type found, possible to be deleted, undelete it first before updating
+                dictionaryPersistenceService.undeleteType(typeId);
                 dictionaryService.updateTypeValue(typeId, type);
             } else { // add the type, no match was found
                 var returnedType = dictionaryService.addType(type);
@@ -315,11 +342,26 @@ public class DossierTemplateImportService {
             this.addEntries(request.getFalsePositives(), typeId, type.getTypeId(), DictionaryEntryType.FALSE_POSITIVE);
             this.addEntries(request.getFalseRecommendations(), typeId, type.getTypeId(), DictionaryEntryType.FALSE_RECOMMENDATION);
         }
-        // remove additional types
+        // remove additional types and not included in the archive
-        Set<String> typesToRemove = currentTypesId.stream().filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet());
+        this.deleteTypes(currentTypes, typeIdsAdded);
-        typesToRemove.forEach(dictionaryService::deleteType);
     }
 
+    private void deleteTypes(List<TypeEntity> currentTypes, Set<String> typeIdsAdded) {
+        Set<String> currentTypesIdSystemManaged = currentTypes.stream().filter(TypeEntity::isSystemManaged).map(TypeEntity::getId).collect(Collectors.toSet());
+
+        // for types system managed just delete the entries
+        Set<String> entriesToRemove = currentTypesIdSystemManaged.stream().filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet());
+        entriesToRemove.forEach(typeId -> {
+            var currentVersion = dictionaryService.getCurrentVersion(typeId);
+            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
+            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
+            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
+            dictionaryPersistenceService.incrementVersion(typeId);
+            typeIdsAdded.add(typeId); // added to the list, since the type can not be deleted
+        });
+        Set<String> typesToRemove = currentTypes.stream().map(TypeEntity::getId).filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet());
+        typesToRemove.forEach(dictionaryService::deleteType);
+    }
     private void updateDossierStates(ImportTemplateResult request, String dossierTemplateId) {
 
         List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
@@ -358,7 +400,7 @@ public class DossierTemplateImportService {
         });
         // remove existing dossier attributes and not included in archive
         Set<String> attributesToRemove = currentDossierAttributeIds.stream().filter(da -> !dossierAttributeIdsAdded.contains(da)).collect(Collectors.toSet());
-        attributesToRemove.forEach(daId -> dossierAttributeConfigPersistenceService.deleteDossierAttribute(daId));
+        attributesToRemove.forEach(dossierAttributeConfigPersistenceService::deleteDossierAttribute);
     }
 
     private void updateFileAttributes(ImportTemplateResult request, String dossierTemplateId) {
@@ -373,7 +415,7 @@ public class DossierTemplateImportService {
         });
         // remove existing file attributes and not included in archive
         Set<String> attributesToRemove = currentFileAttributeIds.stream().filter(fa -> !fileAttributeIdsAdded.contains(fa)).collect(Collectors.toSet());
-        attributesToRemove.forEach(faId -> fileAttributeConfigPersistenceService.deleteFileAttribute(faId));
+        attributesToRemove.forEach(fileAttributeConfigPersistenceService::deleteFileAttribute);
     }
 
     public ImportTemplateResult handleArchive(ImportDossierTemplateRequest request) {
@@ -474,17 +516,17 @@ public class DossierTemplateImportService {
                         } else if (ze.getName().contains(ExportFilename.ENTRIES.getFilename())) {
 
                             List<String> entries = this.readEntries(bytes);
-                            typeEntriesMap.put(this.getTypeId(ze.getName()), entries);
+                            typeEntriesMap.put(this.getType(ze.getName()), entries);
 
                         } else if (ze.getName().contains(ExportFilename.FALSE_POSITIVES.getFilename())) {
 
                             List<String> falsePositives = this.readEntries(bytes);
-                            typeFalsePositivesMap.put(this.getTypeId(ze.getName()), falsePositives);
+                            typeFalsePositivesMap.put(this.getType(ze.getName()), falsePositives);
 
                         } else if (ze.getName().contains(ExportFilename.FALSE_RECOMMENDATION.getFilename())) {
 
                             List<String> falseRecommendations = this.readEntries(bytes);
-                            typeFalseRecommendationsMap.put(this.getTypeId(ze.getName()), falseRecommendations);
+                            typeFalseRecommendationsMap.put(this.getType(ze.getName()), falseRecommendations);
 
                         } else if (ze.getName().contains(ExportFilename.REPORT_TEMPLATE.getFilename())) {
                             var reportTemplateList = objectMapper.readValue(bytes, new TypeReference<List<ReportTemplate>>() {
@@ -554,7 +596,7 @@ public class DossierTemplateImportService {
         return entries;
     }
 
-    private String getTypeId(String filename) {
+    private String getType(String filename) {
         var index = filename.indexOf('/');
         return filename.substring(0, index);
     }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java

@@ -139,10 +139,14 @@ public class DossierTemplateExportService {
             // for every type 1 folder with:
             // 1 json file containing meta info of the type
             // and 1 txt file for every type: entries, false positives and false recommendation
-            var dossierTypes = dossierTemplate.getDossierTypes();
+
+            // remove the types related to dossiers and also the ones that are deleted
+            var dossierTypes = dossierTemplate.getDossierTypes().stream()
+                    .filter(t -> t.getDossierId() == null)
+                    .filter(t -> !t.isDeleted()).collect(Collectors.toList());
             for (TypeEntity typeEntity : dossierTypes) {
-//                log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId());
+//                log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId() + " " + typeEntity.isDeleted());
-                fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getId(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT),
+                fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getType(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT),
                         objectMapper.writeValueAsBytes(convert(typeEntity, Type.class))));
 
                 var entriesValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.ENTRY, null)

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/DownloadCleanupJob.java

@@ -56,7 +56,7 @@ public class DownloadCleanupJob implements Job {
                     log.info("2. Deleting download status {} because DownloadCleanupNotDownloadFilesHours is {} and c+h {} is after {}", downloadStatus, applicationConfigurationEntity.getDownloadCleanupNotDownloadFilesHours(), downloadStatus.getCreationDate()
                             .plusHours(applicationConfigurationEntity.getDownloadCleanupNotDownloadFilesHours()), now);
                     deleteDownload(downloadStatus);
-                } else if (dossierService.getDossierById(dossier.getId()).getSoftDeletedTime() != null) {
+                } else if (dossier != null && dossierService.getDossierById(dossier.getId()).getSoftDeletedTime() != null) {
                     log.info("3. Deleting download {}, because dossier does not exist", downloadStatus.getStorageId());
                     deleteDownload(downloadStatus);
                 }