package com.iqser.red.service.persistence.management.v1.processor.service;

import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.FileAttributesGeneralConfiguration;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ReportTemplateUploadRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ExportFilename;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportTemplateResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
@RequiredArgsConstructor
public class DossierTemplateImportService {

    private static final int THRESHOLD_ENTRIES = 10000;
    private static final int THRESHOLD_SIZE = 1000000000; // 1 GB

    private final DossierTemplateRepository dossierTemplateRepository;
    private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
    private final RulesPersistenceService rulesPersistenceService;
    private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
    private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
    private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
    private final ColorsService colorsService;
    private final DictionaryPersistenceService dictionaryPersistenceService;
    private final DossierStatusPersistenceService dossierStatusPersistenceService;
    private final WatermarkService watermarkService;
    private final ReportTemplateService reportTemplateService;
    private final DictionaryManagementService dictionaryManagementService;
    private final EntryPersistenceService entryPersistenceService;
    private final ReportTemplatePersistenceService reportTemplatePersistenceService;
    private final RulesValidationService rulesValidationService;
    private final StorageService storageService;
    private final ObjectMapper objectMapper = new ObjectMapper();
    private final FileManagementServiceSettings settings;


    public String importDossierTemplate(ImportDossierTemplateRequest request) {

        ImportTemplateResult archiveResult = this.handleArchive(request);

        return this.importDossierTemplate(archiveResult);
    }


    private ImportTemplateResult handleArchive(ImportDossierTemplateRequest request) {

        objectMapper.registerModule(new JavaTimeModule());
        File tempFile;
        try {
            tempFile = FileUtils.createTempFile(request.getArchive(), UUID.randomUUID().toString(), ".zip");
            var importTemplateResult = new ImportTemplateResult();
            importTemplateResult.setDossierTemplateId(request.getDossierTemplateId());
            importTemplateResult.setUserId(request.getUserId());
            importTemplateResult.setUpdateExistingTemplate(request.isUpdateExistingDossierTemplate());

            try (FileInputStream fis = new FileInputStream(tempFile); BufferedInputStream bis = new BufferedInputStream(fis); ZipArchiveInputStream zis = new ZipArchiveInputStream(
                    bis,
                    "UTF8",
                    true,
                    true)) {

                int totalSizeArchive = 0;
                int totalEntryArchive = 0;
                ZipArchiveEntry ze;
                Map<String, ReportTemplate> reportTemplateMap = new HashMap<>();
                Map<String, ByteArrayOutputStream> reportTemplateBytesMap = new HashMap<>();
                List<String> reportTemplateFilenameList = new ArrayList<>();
                Map<String, List<String>> typeEntriesMap = new HashMap<>();
                Map<String, List<String>> typeFalsePositivesMap = new HashMap<>();
                Map<String, List<String>> typeFalseRecommendationsMap = new HashMap<>();

                while ((ze = zis.getNextZipEntry()) != null) {
                    log.debug("---> " + ze.getName() + "  ----  " + ze.isDirectory());
                    totalEntryArchive++;

                    if (!zis.canReadEntryData(ze)) {
                        continue;
                    }
                    if (ze.isUnixSymlink()) {
                        continue;
                    }
                    if (!ze.isDirectory()) {
                        var bos = new ByteArrayOutputStream();
                        var buffer = new byte[2048];
                        var nBytes = 0;
                        int totalSizeEntry = 0;

                        while ((nBytes = zis.read(buffer)) > 0) {
                            bos.write(buffer, 0, nBytes);
                            totalSizeEntry += nBytes;
                            totalSizeArchive += nBytes;

                            double compressionRatio = (float) totalSizeEntry / ze.getCompressedSize();
                            if (compressionRatio > settings.getCompressionThresholdRatio()) {
                                log.debug("zip entry: " + ze.getName() + " - totalSizeEntry: " + totalSizeEntry + " ze.getCompressedSize(): " + ze.getCompressedSize() + " compressionRatio: " + compressionRatio);
                                // ratio between compressed and uncompressed data is highly suspicious, looks like a Zip Bomb Attack
                                throw new BadRequestException("ZIP-Bomb detected (compressionRatio).");
                            }
                        }
                        if (totalSizeArchive > THRESHOLD_SIZE) {
                            log.debug("zip entry: " + ze.getName() + " totalSizeEntry: " + totalSizeArchive);

                            // the uncompressed data size is too much for the application resource capacity
                            throw new BadRequestException("ZIP-Bomb detected. (threshold size)");
                        }

                        if (totalEntryArchive > THRESHOLD_ENTRIES) {
                            log.debug("zip entry: " + ze.getName() + " totalEntryArchive: " + totalEntryArchive);
                            // too much entries in this archive, can lead to inodes exhaustion of the system
                            throw new BadRequestException("ZIP-Bomb detected (threshold entries).");
                        }

                        var bytes = bos.toByteArray();
                        if (ze.getName().contains(ExportFilename.DOSSIER_TEMPLATE_META.getFilename())) {
                            DossierTemplate dossierTemplate = objectMapper.readValue(bytes, DossierTemplate.class);
                            importTemplateResult.setDossierTemplate(dossierTemplate);
                        } else if (ze.getName().contains(ExportFilename.WATERMARK.getFilename())) {
                            List<WatermarkModel> watermarkList = objectMapper.readValue(bytes, new TypeReference<>() {
                            });
                            importTemplateResult.getWatermarks().addAll(watermarkList);
                        } else if (ze.getName().contains(ExportFilename.COLORS.getFilename())) {
                            Colors colors = objectMapper.readValue(bytes, Colors.class);
                            importTemplateResult.setColors(colors);
                        } else if (ze.getName().contains(ExportFilename.DOSSIER_STATUS.getFilename())) {
                            List<DossierStatusInfo> dossierStatusInfoList = objectMapper.readValue(bytes, new TypeReference<>() {
                            });
                            importTemplateResult.getDossierStatusInfos().addAll(dossierStatusInfoList);
                        } else if (ze.getName().contains(ExportFilename.DOSSIER_ATTRIBUTES_CONFIG.getFilename())) {
                            List<DossierAttributeConfig> dossierAttributeConfigs = objectMapper.readValue(bytes, new TypeReference<>() {
                            });
                            importTemplateResult.getDossierAttributesConfigs().addAll(dossierAttributeConfigs);
                        } else if (ze.getName().contains(ExportFilename.FILE_ATTRIBUTE_CONFIG.getFilename())) {
                            List<FileAttributeConfig> fileAttributeConfigs = objectMapper.readValue(bytes, new TypeReference<>() {
                            });
                            importTemplateResult.getFileAttributesConfigs().addAll(fileAttributeConfigs);
                        } else if (ze.getName().contains(ExportFilename.LEGAL_BASIS.getFilename())) {
                            List<LegalBasis> legalBasisList = objectMapper.readValue(bytes, new TypeReference<>() {
                            });
                            importTemplateResult.getLegalBases().addAll(legalBasisList);
                        } else if (ze.getName().contains(ExportFilename.FILE_ATTRIBUTE_GENERAL_CONFIG.getFilename())) {
                            FileAttributesGeneralConfiguration fileAttributesGeneralConfiguration = objectMapper.readValue(bytes, FileAttributesGeneralConfiguration.class);
                            importTemplateResult.setFileAttributesGeneralConfiguration(fileAttributesGeneralConfiguration);
                        } else if (ze.getName().contains(ExportFilename.RULES.getFilename())) {
                            String rules = objectMapper.readValue(bytes, String.class);
                            importTemplateResult.setRuleSet(rules);
                        } else if (ze.getName().contains(ExportFilename.COMPONENT_RULES.getFilename())) {
                            String rules = objectMapper.readValue(bytes, String.class);
                            importTemplateResult.setComponentRuleSet(rules);
                        } else if (ze.getName().contains(ExportFilename.DOSSIER_TYPE.getFilename())) {
                            Type type = objectMapper.readValue(bytes, Type.class);
                            importTemplateResult.getTypes().add(type);
                        } else if (ze.getName().contains(ExportFilename.ENTRIES.getFilename())) {

                            List<String> entries = this.readEntries(bytes);
                            typeEntriesMap.put(this.getType(ze.getName()), entries);

                        } else if (ze.getName().contains(ExportFilename.FALSE_POSITIVES.getFilename())) {

                            List<String> falsePositives = this.readEntries(bytes);
                            typeFalsePositivesMap.put(this.getType(ze.getName()), falsePositives);

                        } else if (ze.getName().contains(ExportFilename.FALSE_RECOMMENDATION.getFilename())) {

                            List<String> falseRecommendations = this.readEntries(bytes);
                            typeFalseRecommendationsMap.put(this.getType(ze.getName()), falseRecommendations);

                        } else if (ze.getName().contains(ExportFilename.REPORT_TEMPLATE.getFilename())) {
                            var reportTemplateList = objectMapper.readValue(bytes, new TypeReference<List<ReportTemplate>>() {
                            });
                            reportTemplateMap = reportTemplateList.stream()
                                    .collect(Collectors.toMap(rt -> rt.isMultiFileReport() ? rt.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : rt.getFileName(),
                                            Function.identity()));
                            reportTemplateFilenameList = reportTemplateList.stream()
                                    .map(rt -> rt.isMultiFileReport() ? rt.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : rt.getFileName())
                                    .collect(Collectors.toList());
                        } else {
                            reportTemplateBytesMap.put(ze.getName(), bos);
                        }
                        bos.close();
                    }
                }
                importTemplateResult.setEntries(typeEntriesMap);
                importTemplateResult.setFalsePositives(typeFalsePositivesMap);
                importTemplateResult.setFalseRecommendations(typeFalseRecommendationsMap);

                for (var reportZe : reportTemplateBytesMap.entrySet()) {
                    if (reportTemplateFilenameList.contains(reportZe.getKey())) {
                        var report = reportTemplateMap.get(reportZe.getKey());
                        importTemplateResult.getReportTemplateUploadRequests()
                                .add(ReportTemplateUploadRequest.builder()
                                        .fileName(report.getFileName())
                                        .activeByDefault(report.isActiveByDefault())
                                        .multiFileReport(report.isMultiFileReport())
                                        .dossierTemplateId(request.getDossierTemplateId() != null ? request.getDossierTemplateId() : "")
                                        .template(reportZe.getValue().toByteArray())
                                        .build());

                    }
                }
                return importTemplateResult;
            } finally {
                if (tempFile != null) {
                    boolean isDeleted = tempFile.delete();
                    if (!isDeleted) {
                        log.debug("tempFile could not be deleted");
                    }
                }
            }
        } catch (IOException | BadRequestException e) {
            throw new BadRequestException(e.getMessage(), e);
        }
    }


    private String importDossierTemplate(ImportTemplateResult request) {

        long start = System.currentTimeMillis();
        String dossierTemplateId;
        var dossierTemplateMeta = request.getDossierTemplate();

        DossierTemplateEntity existingDossierTemplate = null;
        if (request.getDossierTemplateId() != null && request.isUpdateExistingTemplate()) {
            var dossierTemplateOptional = dossierTemplateRepository.findByIdAndNotDeleted(request.getDossierTemplateId());
            existingDossierTemplate = dossierTemplateOptional.orElse(null);
        }

        if (existingDossierTemplate != null) {
            dossierTemplateId = existingDossierTemplate.getId();

            // override the existing dossier template
            updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId());
            dossierTemplateRepository.save(existingDossierTemplate);

            // set rules
            setRulesWhenCompiled(request, dossierTemplateId);

            existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(existingDossierTemplate)
                    .name()));

            // set colors
            this.setColors(dossierTemplateId, request.getColors());

            // set watermarks
            if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
                Set<String> toSetWatermarks = request.getWatermarks().stream().map(WatermarkModel::getName).filter(Objects::nonNull).collect(Collectors.toSet());
                var currentWatermarkConfigs = watermarkService.getWatermarksForDossierTemplateId(dossierTemplateId);
                Set<WatermarkEntity> configsToRemove = currentWatermarkConfigs.stream().filter(c -> !toSetWatermarks.contains(c.getName())).collect(Collectors.toSet());
                var watermarkNameToEntity = currentWatermarkConfigs.stream().collect(Collectors.toMap(WatermarkEntity::getName, Function.identity()));

                request.getWatermarks().forEach(watermark -> {
                    log.info("watermark to add: " + watermark.getName());
                    if (!watermarkNameToEntity.isEmpty() && watermarkNameToEntity.get(watermark.getName()) != null) {

                        watermark.setId(watermarkNameToEntity.get(watermark.getName()).getId());
                    } else {
                        watermark.setId(null);
                    }
                    watermark.setDossierTemplateId(dossierTemplateId);
                    watermarkService.createOrUpdateWatermark(watermark);
                });
                configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId()));
            }

            // dossier status
            if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
                this.updateDossierStates(request, dossierTemplateId);
            } else { // no states to add, delete current states
                List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
                currentStates.forEach(state -> dossierStatusPersistenceService.deleteDossierStatus(state.getId(), null));
            }

            // update dossier attributes
            if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
                this.updateDossierAttributes(request, dossierTemplateId);
            } else { // no dossier attributes to add, but delete existing ones
                var currentConfigs = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
                currentConfigs.forEach(da -> dossierAttributeConfigPersistenceService.deleteDossierAttribute(da.getId()));
            }

            //update file attributes
            if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
                this.updateFileAttributes(request, dossierTemplateId);
            } else { // no file attributes to add, but delete existing
                var currentConfigs = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
                currentConfigs.forEach(fa -> fileAttributeConfigPersistenceService.deleteFileAttribute(fa.getId()));
            }

            // update the types
            if (CollectionUtils.isNotEmpty(request.getTypes())) {
                this.updateTypes(request, dossierTemplateId);
            } else { // no types to add, but remove existing ones
                List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getTypesForDossierTemplate(dossierTemplateId);
                this.deleteTypes(currentTypes, new HashSet<>());
            }

            //set report templates
            var existingReports = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplateId);

            List<String> reportsUpdated = new ArrayList<>();
            if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
                request.getReportTemplateUploadRequests().forEach(reportRequest -> {
                    reportRequest.setDossierTemplateId(dossierTemplateId);
                    var report = reportTemplateService.uploadTemplate(reportRequest);
                    reportsUpdated.add(report.getTemplateId());
                });
            }

            // delete the reports that were not in the import
            existingReports.forEach(r -> {
                String storageId = r.getStorageId();
                if (!reportsUpdated.contains(r.getTemplateId())) {
                    storageService.deleteObject(TenantContext.getTenantId(), storageId);
                    reportTemplatePersistenceService.delete(r.getTemplateId());
                }
            });

            // set legal basis
            if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
                legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
            } else { // delete existing
                legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId);
            }

        } else {
            // creates new dossier template
            if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
                throw new ConflictException("DossierTemplate name must be set");
            }
            this.validateDossierTemplateName(dossierTemplateMeta);
            DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();

            // order is important
            BeanUtils.copyProperties(dossierTemplateMeta, dossierTemplateEntity);
            dossierTemplateEntity.setId(UUID.randomUUID().toString());
            dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
            dossierTemplateEntity.setCreatedBy(request.getUserId());
            //set rules
            setRulesWhenCompiled(request, dossierTemplateEntity.getId());

            var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity);
            loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate));
            dossierTemplateId = loadedDossierTemplate.getId();

            // set colors
            this.setColors(dossierTemplateId, request.getColors());

            // set watermarks
            if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
                request.getWatermarks().forEach(watermark -> {
                    watermark.setDossierTemplateId(dossierTemplateId);
                    watermark.setId(null);
                    watermarkService.createOrUpdateWatermark(watermark);
                });
            }

            // dossier status
            if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
                request.getDossierStatusInfos().forEach(state -> {
                    state.setId(null);
                    this.updateDossierStatus(dossierTemplateId, state);
                });
            }

            //set dossier attributes
            if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
                request.getDossierAttributesConfigs().forEach(da -> {
                    da.setId(null); // in order to create a new dossier attribute
                    dossierAttributeConfigPersistenceService.addOrUpdateDossierAttribute(dossierTemplateId, convert(da, DossierAttributeConfigEntity.class));
                });
            }

            //set file attributes
            if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
                request.getFileAttributesConfigs().forEach(fa -> {
                    fa.setId(null); // in order to force creation of new file attribute
                    fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId, convert(fa, FileAttributeConfigEntity.class));
                });
            }

            //set types
            if (CollectionUtils.isNotEmpty(request.getTypes())) {
                for (var type : request.getTypes()) {
                    type.setDossierTemplateId(dossierTemplateId);
                    var returnedType = dictionaryManagementService.addType(type);
                    this.addEntries(request.getEntries(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.ENTRY);
                    this.addEntries(request.getFalsePositives(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_POSITIVE);
                    this.addEntries(request.getFalseRecommendations(), returnedType.getTypeId(), returnedType.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);

                }
            }

            //set report templates
            if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
                request.getReportTemplateUploadRequests().forEach(reportRequest -> {
                    reportRequest.setDossierTemplateId(dossierTemplateId);
                    reportTemplateService.uploadTemplate(reportRequest);
                });
            }
            // set legal basis
            if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
                legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
            }
        }

        // file attribute general configuration
        if (request.getFileAttributesGeneralConfiguration() != null) {
            fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
                    convert(request.getFileAttributesGeneralConfiguration(), FileAttributesGeneralConfigurationEntity.class));
        }

        long elapsedTime = System.currentTimeMillis() - start;
        log.info("stop import dossier template elapsedTime: " + elapsedTime + "for: " + dossierTemplateId);
        return dossierTemplateId;
    }


    private void setRulesWhenCompiled(ImportTemplateResult request, String dossierTemplateEntity) {

        DroolsSyntaxValidation droolsSyntaxValidation = rulesValidationService.validateRules(RuleFileType.ENTITY, request.getRuleSet());
        if (!droolsSyntaxValidation.isCompiled()) {
            droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(errorMessage -> log.error(errorMessage.getMessage()));
            throw new BadRequestException("The entity rules do not compile!");
        }
        rulesPersistenceService.setRules(request.getRuleSet(), dossierTemplateEntity, RuleFileType.ENTITY);
        if (request.getComponentRuleSet() != null) {
            DroolsSyntaxValidation componentDroolsSyntaxValidation = rulesValidationService.validateRules(RuleFileType.COMPONENT, request.getComponentRuleSet());
            if (!componentDroolsSyntaxValidation.isCompiled()) {
                componentDroolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(errorMessage -> log.error(errorMessage.getMessage()));
                throw new BadRequestException("The component rules do not compile!");
            }
            rulesPersistenceService.setRules(request.getComponentRuleSet(), dossierTemplateEntity, RuleFileType.COMPONENT);
        }
    }


    private List<String> readEntries(byte[] bytes) {

        ByteArrayInputStream bInput = new ByteArrayInputStream(bytes);
        DataInputStream in = new DataInputStream(bInput);
        List<String> entries = new ArrayList<>();
        try {
            while (in.available() > 0) {
                String entry = in.readUTF();
                entries.add(entry);

            }
        } catch (IOException e) {
            log.debug("exception: ", e);
            throw new BadRequestException("Error while reading the entries", e);
        }
        return entries;
    }


    private String getType(String filename) {

        var index = filename.indexOf('/');
        return filename.substring(0, index);
    }


    private void updateDossierTemplateMeta(DossierTemplateEntity dossierTemplateEntity, DossierTemplate dossierTemplate, String userId) {

        if (!dossierTemplateEntity.getName().equalsIgnoreCase(dossierTemplate.getName())) {
            this.validateDossierTemplateName(dossierTemplate);
        }

        BeanUtils.copyProperties(dossierTemplate, dossierTemplateEntity, "modifiedBy", "dateModified", "validFrom", "validTo", "downloadFileTypes");
        OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
        dossierTemplateEntity.setDateModified(now);
        dossierTemplateEntity.setModifiedBy(userId);
        dossierTemplateEntity.setValidFrom(dossierTemplate.getValidFrom() == null ? dossierTemplateEntity.getValidFrom() : dossierTemplate.getValidFrom());
        dossierTemplateEntity.setValidTo(dossierTemplate.getValidTo() == null ? dossierTemplateEntity.getValidTo() : dossierTemplate.getValidTo());

        dossierTemplateEntity.setDownloadFileTypes(dossierTemplate.getDownloadFileTypes() == null || dossierTemplate.getDownloadFileTypes()
                .isEmpty() ? dossierTemplateEntity.getDownloadFileTypes() : dossierTemplate.getDownloadFileTypes());

    }


    private void setColors(String dossierTemplateId, Colors requestedColors) {
        // set colors
        if (requestedColors != null) {
            ColorsEntity colorsEntity = convert(requestedColors, ColorsEntity.class);
            colorsEntity.setDossierTemplateId(dossierTemplateId);
            colorsService.saveColors(colorsEntity);
        }
    }


    private void updateDossierStates(ImportTemplateResult request, String dossierTemplateId) {

        List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);

        Set<String> currentStatesId = currentStates.stream().map(DossierStatusInfo::getId).collect(Collectors.toSet());
        Set<String> stateIdsAdded = new HashSet<>();
        var stateToEntityMap = currentStates.stream().collect(Collectors.toMap(DossierStatusInfo::getName, Function.identity()));
        for (var state : request.getDossierStatusInfos()) {
            state.setDossierTemplateId(dossierTemplateId);
            String stateId;
            if (currentStatesId.contains(state.getId())) { //check by id
                stateId = state.getId();
            } else if (!stateToEntityMap.isEmpty() && stateToEntityMap.get(state.getName()) != null) { //check by name
                state.setId(stateToEntityMap.get(state.getName()).getId()); // force the id found to update the dossier status
            } else { // add the state, no match was found
                state.setId(null); // force null to create the dossier status
            }
            stateId = this.updateDossierStatus(dossierTemplateId, state).getId();
            stateIdsAdded.add(stateId);

        }
        // remove additional states
        Set<String> statesToRemove = currentStatesId.stream().filter(t -> !stateIdsAdded.contains(t)).collect(Collectors.toSet());
        statesToRemove.forEach(stateId -> dossierStatusPersistenceService.deleteDossierStatus(stateId, null));
    }


    private void updateDossierAttributes(ImportTemplateResult request, String dossierTemplateId) {

        List<DossierAttributeConfigEntity> currentFileAttributes = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
        Set<String> currentDossierAttributeIds = currentFileAttributes.stream().map(DossierAttributeConfigEntity::getId).collect(Collectors.toSet());
        Set<String> dossierAttributeIdsAdded = new HashSet<>();

        request.getDossierAttributesConfigs().forEach(da -> {
            da.setId(dossierAttributeConfigPersistenceService.getDossierAttributeByIdOrName(dossierTemplateId, da.getId(), da.getLabel()));
            DossierAttributeConfigEntity result = dossierAttributeConfigPersistenceService.addOrUpdateDossierAttribute(dossierTemplateId,
                    convert(da, DossierAttributeConfigEntity.class));
            dossierAttributeIdsAdded.add(result.getId());
        });
        // remove existing dossier attributes and not included in archive
        Set<String> attributesToRemove = currentDossierAttributeIds.stream().filter(da -> !dossierAttributeIdsAdded.contains(da)).collect(Collectors.toSet());
        attributesToRemove.forEach(dossierAttributeConfigPersistenceService::deleteDossierAttribute);
    }


    private void updateFileAttributes(ImportTemplateResult request, String dossierTemplateId) {

        List<FileAttributeConfigEntity> currentFileAttributes = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
        Set<String> currentFileAttributeIds = currentFileAttributes.stream().map(FileAttributeConfigEntity::getId).collect(Collectors.toSet());
        Set<String> fileAttributeIdsAdded = new HashSet<>();
        request.getFileAttributesConfigs().forEach(fa -> {
            //check by id or by dossierTemplate and label and set the corresponding Id if found
            fa.setId(fileAttributeConfigPersistenceService.getFileAttributeByIdOrName(dossierTemplateId, fa.getId(), fa.getLabel()));
            FileAttributeConfigEntity fileAttributeAdded = fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId,
                    convert(fa, FileAttributeConfigEntity.class));
            fileAttributeIdsAdded.add(fileAttributeAdded.getId());
            if (fa.isPrimaryAttribute()) {
                fileAttributeConfigPersistenceService.updatePrimaryAttribute(fileAttributeAdded.getId());
            }
        });
        // remove existing file attributes and not included in archive
        Set<String> attributesToRemove = currentFileAttributeIds.stream().filter(fa -> !fileAttributeIdsAdded.contains(fa)).collect(Collectors.toSet());
        attributesToRemove.forEach(fileAttributeConfigPersistenceService::deleteFileAttribute);
    }


    private void updateTypes(ImportTemplateResult request, String dossierTemplateId) {

        List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getTypesForDossierTemplate(dossierTemplateId)
                .stream()
                .filter(t -> t.getDossierId() == null)
                .collect(Collectors.toList());

        Set<String> currentTypesId = currentTypes.stream().map(TypeEntity::getId).collect(Collectors.toSet());
        Set<String> typeIdsAdded = new HashSet<>();
        var typeToEntityMap = currentTypes.stream().collect(Collectors.toMap(TypeEntity::getType, Function.identity()));
        for (var type : request.getTypes()) {
//            log.info("Adding type: " + type.getType() + " id: " + type.getId());
            type.setDossierTemplateId(dossierTemplateId);
            String typeId;
            if (currentTypesId.contains(type.getId())) { //check by id
                typeId = type.getId();
                // type found, possible to be deleted, undelete it first before updating
                dictionaryPersistenceService.undeleteType(typeId);
                dictionaryManagementService.updateTypeValue(typeId, type);
            } else if (!typeToEntityMap.isEmpty() && typeToEntityMap.get(type.getType()) != null) { //check by name
                typeId = typeToEntityMap.get(type.getType()).getId();
                // type found, possible to be deleted, undelete it first before updating
                dictionaryPersistenceService.undeleteType(typeId);
                dictionaryManagementService.updateTypeValue(typeId, type);
            } else { // add the type, no match was found
                var returnedType = dictionaryManagementService.addType(type);
                typeId = returnedType.getTypeId();
            }
            typeIdsAdded.add(typeId);

            this.addEntries(request.getEntries(), typeId, type.getType(), DictionaryEntryType.ENTRY);
            this.addEntries(request.getFalsePositives(), typeId, type.getType(), DictionaryEntryType.FALSE_POSITIVE);
            this.addEntries(request.getFalseRecommendations(), typeId, type.getType(), DictionaryEntryType.FALSE_RECOMMENDATION);
        }
        // remove additional types and not included in the archive
        this.deleteTypes(currentTypes, typeIdsAdded);
    }


    private void deleteTypes(List<TypeEntity> currentTypes, Set<String> typeIdsAdded) {

        Set<String> currentTypesIdSystemManaged = currentTypes.stream().filter(TypeEntity::isSystemManaged).map(TypeEntity::getId).collect(Collectors.toSet());

        // for types system managed just delete the entries
        Set<String> entriesToRemove = currentTypesIdSystemManaged.stream().filter(t -> !typeIdsAdded.contains(t)).collect(Collectors.toSet());
        entriesToRemove.forEach(typeId -> {
            var currentVersion = dictionaryManagementService.getCurrentVersion(typeId);
            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
            dictionaryPersistenceService.incrementVersion(typeId);
            typeIdsAdded.add(typeId); // added to the list, since the type can not be deleted
        });
        Set<String> typesToRemove = currentTypes.stream().filter(t -> !t.isDeleted()) // remove the ones already soft deleted
                .map(TypeEntity::getId).filter(t -> !typeIdsAdded.contains(t)) // exclude the type ids already added from the import
                .filter(t -> !currentTypesIdSystemManaged.contains(t)) // exclude the types system managed
                .collect(Collectors.toSet());
        typesToRemove.forEach(dictionaryManagementService::deleteType);
    }


    private void validateDossierTemplateName(DossierTemplate dossierTemplateMeta) {

        int nameSuffix = 0;
        String dossierTemplateName = dossierTemplateMeta.getName();
        while (dossierTemplatePersistenceService.isDossierTemplateNameNotUnique(dossierTemplateMeta.getName())) {
            if (nameSuffix == 0) {
                dossierTemplateMeta.setName("Copy of " + dossierTemplateName);
            } else {
                dossierTemplateMeta.setName("Copy of " + dossierTemplateName + " - " + nameSuffix);
            }
            nameSuffix++;
        }
    }


    private DossierStatusEntity updateDossierStatus(String dossierTemplateId, DossierStatusInfo state) {

        var dossierStatusRequest = CreateOrUpdateDossierStatusRequest.builder()
                .dossierStatusId(state.getId())
                .name(state.getName())
                .description(state.getDescription())
                .color(state.getColor())
                .dossierTemplateId(dossierTemplateId)
                .rank(state.getRank())
                .build();
        return dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
    }


    private void addEntries(Map<String, List<String>> entries, String typeId, String typeName, DictionaryEntryType dictionaryType) {

        if (entries != null && !entries.isEmpty() && entries.get(typeName) != null && !entries.get(typeName).isEmpty()) {
            dictionaryManagementService.addEntries(typeId, entries.get(typeName), true, true, dictionaryType);
        } else { // no entries, delete current entries
            List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryType, null).stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList());
            if (!existing.isEmpty()) {
                dictionaryManagementService.deleteEntries(typeId, existing, dictionaryType);
            }
        }
    }

}
