Pull request #687: RED-6720

Merge in RED/persistence-service from RED-6720 to master

* commit 'c6045189e6b964a3e44d98759bc54c461649bd06':
  RED-6270: Changed data access to avoid lazy loading, making it possible to run the dossier-template import outside an enclosing transaction
  RED-6270: Added try-with-resources to correctly close file stream
  RED-6270: Removed transactional annotation on the dossier-template import, since it uses both Hibernate and JDBC (for runtime performance reasons) and otherwise runs into query ordering issues
  RED-6270: Repackaged incorrectly packaged dossier template and created dossier template import test that always runs (it does not have local dependencies)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java

@@ -22,8 +22,6 @@ import java.util.UUID;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import javax.transaction.Transactional;
-
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
@@ -37,7 +35,6 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
@@ -108,7 +105,6 @@ public class DossierTemplateImportService {
     private final FileManagementServiceSettings settings;
 
 
-    @Transactional
     public String importDossierTemplate(ImportDossierTemplateRequest request) {
 
         ImportTemplateResult archiveResult = this.handleArchive(request);
@@ -360,11 +356,7 @@ public class DossierTemplateImportService {
             if (CollectionUtils.isNotEmpty(request.getTypes())) {
                 this.updateTypes(request, dossierTemplateId);
             } else { // no types to add, but remove existing ones
-                List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId)
+                List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getTypesForDossierTemplate(dossierTemplateId);
-                        .getDossierTypes()
-                        .stream()
-                        .filter(t -> t.getDossierId() == null)
-                        .collect(Collectors.toList());
                 this.deleteTypes(currentTypes, new HashSet<>());
             }
 
@@ -393,11 +385,7 @@ public class DossierTemplateImportService {
             if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
                 legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
             } else { // delete existing
-                var existingLegalBasis = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId)
+                legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId);
-                        .stream()
-                        .map(LegalBasisEntity::getName)
-                        .collect(Collectors.toList());
-                legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId, existingLegalBasis);
             }
 
         } else {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java

@@ -5,6 +5,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ma
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -264,7 +265,11 @@ public class DossierTemplateExportService {
     private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) {
 
         long start = System.currentTimeMillis();
-        fileManagementStorageService.storeObject(storageId, fileSystemBackedArchiver.toInputStream());
+        try (InputStream data = fileSystemBackedArchiver.toInputStream()) {
+            fileManagementStorageService.storeObject(storageId, data);
+        } catch (IOException ex) {
+            log.warn("IO error when sending file to storage", ex);
+        }
         log.info("Successfully stored zip for downloadId {}, took {}", storageId, System.currentTimeMillis() - start);
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierTemplatePersistenceService.java

@@ -12,11 +12,13 @@ import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CreateOrUpdateDossierTemplateRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
 
@@ -32,6 +34,8 @@ public class DossierTemplatePersistenceService {
     private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
     private final RulesPersistenceService rulesPersistenceService;
 
+    private final TypeRepository typeRepository;
+
 
     @Transactional
     public DossierTemplateEntity createOrUpdateDossierTemplate(CreateOrUpdateDossierTemplateRequest createOrUpdateDossierRequest) {
@@ -134,6 +138,13 @@ public class DossierTemplatePersistenceService {
     }
 
 
+    @Transactional
+    public List<TypeEntity> getTypesForDossierTemplate(String dossierTemplateId) {
+
+        return typeRepository.findAllByDossierTemplateIdAndDossierId(dossierTemplateId, null);
+    }
+
+
     @Transactional
     public void deleteDossierTemplate(String dossierTemplateId, String deletingUserId) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/LegalBasisMappingPersistenceService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
 import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -37,6 +38,16 @@ public class LegalBasisMappingPersistenceService {
     }
 
 
+    @Transactional
+    public void deleteLegalBasis(String dossierTemplateId) {
+
+        var mapping = getLegalBasisMappingOrCreate(dossierTemplateId);
+        mapping.setLegalBasis(Collections.emptyList());
+        mapping.setVersion(mapping.getVersion() + 1);
+        legalBasisMappingRepository.save(mapping);
+    }
+
+
     private LegalBasisMappingEntity getLegalBasisMappingOrCreate(String dossierTemplateId) {
 
         return legalBasisMappingRepository.findById(dossierTemplateId).orElseGet(() -> {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java

@@ -37,6 +37,9 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
     List<TypeEntity> findAllTypesByDossierTemplateIdOrDossierId(String dossierTemplateId, String dossierId);
 
 
+    List<TypeEntity> findAllByDossierTemplateIdAndDossierId(String dossierTemplateId, String dossierId);
+
+
     @Query("select coalesce(sum(t.version),0) from TypeEntity t where t.dossierId = :dossierId")
     long getVersionForDossierId(String dossierId);
 
@@ -45,7 +48,13 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
     long getVersionForDossierTemplateId(String dossierTemplateId);
 
 
-    @Query("select new com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionarySummaryResponse(dt.id, t.id, t.type, t.label, count(e)) " + "from DossierTemplateEntity dt " + "left join TypeEntity t on t.dossierTemplateId = dt.id " + "left join DictionaryEntryEntity e on e.typeId = t.id " + "where t.softDeletedTime is null and dt.id in :dossierTemplateIds and t.dossierId is null and (e.id is null or e.deleted = false) " + "group by dt.id, t.id, t.type, t.label ")
+    @Query("""
+            select new com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionarySummaryResponse(dt.id, t.id, t.type, t.label, count(e))
+            from DossierTemplateEntity dt
+            left join TypeEntity t on t.dossierTemplateId = dt.id
+            left join DictionaryEntryEntity e on e.typeId = t.id
+            where t.softDeletedTime is null and dt.id in :dossierTemplateIds and t.dossierId is null and (e.entryId is null or e.deleted = false)
+            group by dt.id, t.id, t.type, t.label""")
     List<DictionarySummaryResponse> findDictionarySummaryList(Set<String> dossierTemplateIds);
 
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportTest.java

@@ -43,8 +43,9 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT
 
     @SneakyThrows
     @Test
+    // For local testing, so disabled for CI
     @Disabled
-    public void testDossierTemplateImport() {
+    public void testLocalDossierTemplateImport() {
 
         var importDir = new File("/Users/timobejan/work/dossier-templates-v2/dev");
         assertThat(importDir).isNotNull();
@@ -55,13 +56,7 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT
             if (file.isDirectory()) {
                 var archive = pack(file.getAbsolutePath());
                 log.info("Importing file: " + file.getName() + " " + " with size: " + archive.length);
-                var request = new ImportDossierTemplateRequest();
+                testDossierTemplateImport(archive);
-                request.setArchive(archive);
-                request.setUpdateExistingDossierTemplate(false);
-                request.setUserId("system");
-                DossierTemplate dossierTemplate = dossierTemplateManagementService.importDossierTemplate(request);
-                assertThat(dossierTemplate).isNotNull();
-                assertThat(dossierTemplate.getId()).isNotBlank();
             }
         }
     }
@@ -93,12 +88,47 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT
     }
 
 
+    @SneakyThrows
+    @Test
+    public void testDossierTemplateImport() {
+
+        TenantContext.setTenantId("redaction");
+
+        var archive = loadFileFromClasspath("EFSA_sanitisation_GFL_v1.zip");
+        testDossierTemplateImport(archive);
+    }
+
+
+    @SneakyThrows
+    private byte[] loadFileFromClasspath(String filename) {
+
+        ClassPathResource classPathResource = new ClassPathResource(DOSSIERTEMPLATES_PATH + filename);
+        assertThat(classPathResource.exists()).isTrue();
+        try (InputStream inputStream = classPathResource.getInputStream()) {
+            return inputStream.readAllBytes();
+        }
+    }
+
+
+    private void testDossierTemplateImport(byte[] archive) {
+
+        var request = new ImportDossierTemplateRequest();
+        request.setArchive(archive);
+        request.setUpdateExistingDossierTemplate(false);
+        request.setUserId("system");
+        DossierTemplate dossierTemplate = dossierTemplateManagementService.importDossierTemplate(request);
+        assertThat(dossierTemplate).isNotNull();
+        assertThat(dossierTemplate.getId()).isNotBlank();
+    }
+
+
     @Test
     @SneakyThrows
+    // This is currently not working, since there seems to be some missing Feign decoder config for tests
     @Disabled
     public void dossierImportClientTest() {
 
-        var multipartFile = loadDossierTemplateFromClasspath("EFSA_sanitisation_GFL_v1.zip");
+        var multipartFile = loadMultiPartFileFromClasspath("EFSA_sanitisation_GFL_v1.zip");
         DossierTemplateModel dossierTemplateModel = dossierTemplateClient.importDossierTemplate(multipartFile, null, false);
         assertThat(dossierTemplateModel).isNotNull();
         assertThat(dossierTemplateModel.getId()).isNotBlank();
@@ -107,7 +137,7 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT
 
 
     @SneakyThrows
-    private MockMultipartFile loadDossierTemplateFromClasspath(String filename) {
+    private MockMultipartFile loadMultiPartFileFromClasspath(String filename) {
 
         ClassPathResource classPathResource = new ClassPathResource(DOSSIERTEMPLATES_PATH + filename);
         assertThat(classPathResource.exists()).isTrue();