From 35f3b3e06c85df7c14d1616bc6a6e9d67d6c0fdf Mon Sep 17 00:00:00 2001 From: devplant Date: Wed, 14 Sep 2022 09:42:06 +0300 Subject: [PATCH] RED-2200 - Dossier Template export and import - add implementation for import and export. - add tests --- .../importexport/ExportDownloadRequest.java | 12 + .../importexport/ExportFilename.java | 29 ++ .../ImportDossierTemplateRequest.java | 18 + .../importexport/ImportTemplateResult.java | 72 +++ .../resources/DossierTemplateResource.java | 15 + .../DownloadStatusPersistenceService.java | 11 +- .../persistence-service-server-v1/pom.xml | 13 + .../configuration/MessagingConfiguration.java | 16 + .../controller/DictionaryController.java | 250 +---------- .../controller/DossierTemplateController.java | 24 + .../controller/ReportTemplateController.java | 36 +- .../v1/server/service/DictionaryService.java | 284 ++++++++++++ .../service/DossierTemplateImportService.java | 410 ++++++++++++++++++ .../server/service/ReportTemplateService.java | 61 +++ .../export/DossierTemplateExportService.java | 207 +++++++++ .../export/ExportDownloadMessageReceiver.java | 30 ++ .../v1/server/utils/StorageIdUtils.java | 5 + .../tests/DossierTemplateTest.java | 222 +++++++++- .../server/integration/tests/DossierTest.java | 4 +- 19 files changed, 1440 insertions(+), 279 deletions(-) create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportDownloadRequest.java create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportFilename.java create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportDossierTemplateRequest.java create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportTemplateResult.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReportTemplateService.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/ExportDownloadMessageReceiver.java diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportDownloadRequest.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportDownloadRequest.java new file mode 100644 index 000000000..8613d179d --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportDownloadRequest.java @@ -0,0 +1,12 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport; + +import lombok.*; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class ExportDownloadRequest { + private String userId; + private String dossierTemplateId; +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportFilename.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportFilename.java new file mode 100644 index 000000000..8389b9ccb --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportFilename.java @@ -0,0 +1,29 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport; + +import lombok.Getter; + +public enum ExportFilename { + + DOSSIER_TEMPLATE_META("dossierTemplate"), + WATERMARK("watermark"), + COLORS("colors"), + DOSSIER_STATUS("dossierStatusList"), + DOSSIER_ATTRIBUTES_CONFIG("dossierAttributesConfigList"), + FILE_ATTRIBUTE_CONFIG("fileAttributeConfigList"), + FILE_ATTRIBUTE_GENERAL_CONFIG("fileAttributeGeneralConfigList"), + LEGAL_BASIS("legalBasisMappingList"), + RULES("rules"), + REPORT_TEMPLATE("reportTemplateList"), + DOSSIER_TYPE("dossierType"), + ENTRIES("entries"), + FALSE_POSITIVES("falsePositives"), + FALSE_RECOMMENDATION("falseRecommendations"); + + + @Getter + private final String filename; + + ExportFilename(String filename) { + this.filename = filename; + } +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportDossierTemplateRequest.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportDossierTemplateRequest.java new file mode 100644 index 000000000..ce8355e8d --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportDossierTemplateRequest.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport; + +import lombok.*; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class ImportDossierTemplateRequest { + @NonNull + private byte[] archive; + + private String dossierTemplateId; + + private String userId; + + private boolean updateExistingDossierTemplate; +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportTemplateResult.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportTemplateResult.java new file mode 100644 index 000000000..01b571a7a --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportTemplateResult.java @@ -0,0 +1,72 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport; + +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.FileAttributesGeneralConfiguration; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUploadRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Watermark; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatusInfo; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImportTemplateResult { + + public String dossierTemplateId; + public String userId; + + public boolean updateExistingTemplate; + + public DossierTemplate dossierTemplate; + + @Builder.Default + public List watermarks = new ArrayList<>(); + + public Colors colors; + + @Builder.Default + public List dossierStatusInfos = new ArrayList<>(); + + @Builder.Default + public List dossierAttributesConfigs = new ArrayList<>(); + + public FileAttributesGeneralConfiguration fileAttributesGeneralConfiguration; + + @Builder.Default + public List fileAttributesConfigs = new ArrayList<>(); + + @Builder.Default + public List legalBases = new ArrayList<>(); + + public String ruleSet; + + @Builder.Default + public List reportTemplateUploadRequests = new ArrayList<>(); + + @Builder.Default + public List types= new ArrayList<>(); + + @Builder.Default + public Map> entries = new HashMap<>(); + + @Builder.Default + public Map> falsePositives = new HashMap<>(); + + @Builder.Default + public Map> falseRecommendations = new HashMap<>(); + +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateResource.java index c80eb3232..5ef3c9ca7 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateResource.java @@ -1,8 +1,12 @@ package com.iqser.red.service.persistence.service.v1.api.resources; +import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CreateOrUpdateDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportDownloadRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportDossierTemplateRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportTemplateResult; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; @@ -19,6 +23,8 @@ public interface DossierTemplateResource { String USER_ID_PARAM = "userId"; String CLONE_PATH = "/clone"; + String EXPORT_PATH = "/export"; + String IMPORT_PATH = "/import"; @ResponseBody @ResponseStatus(HttpStatus.ACCEPTED) @@ -46,4 +52,13 @@ public interface DossierTemplateResource { DossierTemplate cloneDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody CloneDossierTemplateRequest cloneDossierTemplateRequest); + @PostMapping(value = DOSSIER_TEMPLATE_PATH + EXPORT_PATH + "/prepare", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + JSONPrimitive prepareExportDownload(@RequestBody ExportDownloadRequest request); + + @PostMapping(value = DOSSIER_TEMPLATE_PATH + EXPORT_PATH + "/create", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + void createExportDownload(@RequestParam String userId, @RequestParam String storageId); + + @PostMapping(value = DOSSIER_TEMPLATE_PATH + IMPORT_PATH, consumes = MediaType.APPLICATION_JSON_VALUE) + void importDossierTemplate(@RequestBody ImportDossierTemplateRequest request); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java index 0ba2843f6..8fd378010 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -38,12 +39,18 @@ public class DownloadStatusPersistenceService { downloadStatus.setMimeType(mimeType); downloadStatus.setDossier(dossier); downloadStatus.setCreationDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - downloadStatus.setFiles(fileRepository.findAllById(fileIds)); - downloadStatus.setDownloadFileTypes(new HashSet<>(downloadFileTypes)); + downloadStatus.setFiles(fileIds!= null ? fileRepository.findAllById(fileIds) : new ArrayList<>()); + downloadStatus.setDownloadFileTypes(downloadFileTypes != null ?new HashSet<>(downloadFileTypes) : new HashSet<>()); downloadStatusRepository.save(downloadStatus); } + // use this to create a status for export dossier template. + public void createStatus(String userId, String storageId, String filename, String mimeType) { + + this.createStatus(userId, storageId, null, filename, mimeType, null, null); + } + @Transactional public void updateStatus(String storageId, DownloadStatusValue status) { diff --git a/persistence-service-v1/persistence-service-server-v1/pom.xml b/persistence-service-v1/persistence-service-server-v1/pom.xml index d24d90ab4..15ab8a5cd 100644 --- a/persistence-service-v1/persistence-service-server-v1/pom.xml +++ b/persistence-service-v1/persistence-service-server-v1/pom.xml @@ -93,6 +93,19 @@ com.iqser.red.commons spring-commons + + com.iqser.red.commons + jackson-commons + + + org.apache.commons + commons-compress + 1.21 + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + org.postgresql diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java index c0e869bcf..2ee4b3492 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java @@ -18,6 +18,9 @@ public class MessagingConfiguration { public static final String DOWNLOAD_QUEUE = "downloadQueue"; public static final String DOWNLOAD_DLQ = "downloadDLQ"; + public static final String EXPORT_DOWNLOAD_QUEUE = "exportDownloadQueue"; + public static final String EXPORT_DOWNLOAD_DLQ = "exportDownloadDLQ"; + public static final String REPORT_QUEUE = "reportQueue"; public static final String REPORT_DLQ = "reportDLQ"; @@ -159,6 +162,19 @@ public class MessagingConfiguration { return QueueBuilder.durable(DOWNLOAD_DLQ).build(); } + @Bean + public Queue exportDownloadQueue() { + + return QueueBuilder.durable(EXPORT_DOWNLOAD_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", EXPORT_DOWNLOAD_DLQ).build(); + } + + + @Bean + public Queue exportDownloadDeadLetterQueue() { + + return QueueBuilder.durable(EXPORT_DOWNLOAD_DLQ).build(); + } + @Bean public Queue reportQueue() { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java index c76c7240f..6e47feb0d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java @@ -1,13 +1,7 @@ package com.iqser.red.service.peristence.v1.server.controller; -import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities; -import com.iqser.red.service.peristence.v1.server.service.StopwordService; -import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; +import com.iqser.red.service.peristence.v1.server.service.DictionaryService; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; -import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; @@ -18,8 +12,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; @@ -27,15 +19,8 @@ import org.springframework.web.bind.annotation.RestController; import javax.transaction.Transactional; import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; @RestController @RequiredArgsConstructor @@ -45,7 +30,7 @@ public class DictionaryController implements DictionaryResource { private final EntryPersistenceService entryPersistenceService; private final DictionaryPersistenceService dictionaryPersistenceService; private final ColorsService colorsService; - private final StopwordService stopwordService; + private final DictionaryService dictionaryService; @Override @@ -54,177 +39,35 @@ public class DictionaryController implements DictionaryResource { @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries, @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) { - Set cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet()); - - if (CollectionUtils.isEmpty(entries)) { - throw new BadRequestException("Entry list is empty."); - } - - var invalidEntries = getInvalidEntries(cleanEntries); - - if (!ignoreInvalidEntries && CollectionUtils.isNotEmpty(invalidEntries)) { - throw new BadRequestException("Error(s) validating dictionary entries:\\n" + String.join("\\n", invalidEntries)); - } else { - cleanEntries.removeAll(invalidEntries); - } - - // To check whether the type exists, type should not be added into database implicitly by addEntry. - Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); - if (!typeResult.isHasDictionary()) { - throw new BadRequestException("Entity type does not have a dictionary"); - } - - var currentVersion = getCurrentVersion(typeResult); - - if (removeCurrent) { - entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType); - entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType); - } else { - entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType); - } - - dictionaryPersistenceService.incrementVersion(typeId); + dictionaryService.addEntries(typeId, entries, removeCurrent, ignoreInvalidEntries, dictionaryEntryType); } @Override public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List entries, @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) { - // To check whether the type exists - Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); - - var currentVersion = getCurrentVersion(typeResult); - - if (typeResult.isCaseInsensitive()) { - List existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY, null) - .stream() - .map(BaseDictionaryEntry::getValue) - .collect(toList()); - entryPersistenceService.deleteEntries(typeId, existing.stream() - .filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)) - .collect(toList()), currentVersion + 1, dictionaryEntryType); - } else { - entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType); - } - - dictionaryPersistenceService.incrementVersion(typeId); + dictionaryService.deleteEntries(typeId, entries, dictionaryEntryType); } @Override public void updateTypeValue(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody Type typeRequest) { - validateColor(typeRequest.getHexColor()); - validateBoolean(typeRequest.isHint(), "isHint"); - validateBoolean(typeRequest.isCaseInsensitive(), "isCaseInsensitive"); - String skippedHexColor = typeRequest.getSkippedHexColor(); - - if (StringUtils.isBlank(skippedHexColor)) { //use the default value - skippedHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getSkippedColor(); - typeRequest.setSkippedHexColor(skippedHexColor); - } else { - validateColor(skippedHexColor); - } - - String recommendationHexColor = typeRequest.getRecommendationHexColor(); - if (StringUtils.isBlank(recommendationHexColor)) { //use the default value - recommendationHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getRecommendationColor(); - typeRequest.setRecommendationHexColor(recommendationHexColor); - } else { - validateColor(typeRequest.getRecommendationHexColor()); - } - - - // To check whether the type exists - Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); - - if (typeRequest.getLabel() != null) { - checkForDuplicateLabels(typeResult.getDossierTemplateId(), typeResult.getDossierId(), typeResult.getType(), typeRequest.getLabel()); - } else { - typeRequest.setLabel(typeResult.getLabel()); - } - - dictionaryPersistenceService.updateType(typeId, convert(typeRequest, TypeEntity.class)); - - if (typeResult.isHint() != typeRequest.isHint() || typeResult.isCaseInsensitive() != typeRequest.isCaseInsensitive() || typeResult - .getRank() != typeRequest.getRank()) { - - var currentVersion = getCurrentVersion(typeResult); - - entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.ENTRY); - entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); - entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); - - } - + dictionaryService.updateTypeValue(typeId, typeRequest); } @Override public Type addType(@RequestBody Type typeRequest) { - if (typeRequest.getDossierTemplateId() == null) { - throw new BadRequestException("Dossier template id does not exist."); - } - - if (typeRequest.getLabel() == null || typeRequest.getLabel().isEmpty()) { - String label = humanizedDictionaryType(typeRequest.getType()); - typeRequest.setLabel(label); - } - - checkForDuplicateLabels(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), typeRequest.getType(), typeRequest - .getLabel()); - - if (dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), false) - .stream() - .anyMatch(typeResult -> typeRequest.getDossierId() != null && typeResult.getDossierId() != null && typeRequest - .getDossierId() - .equals(typeResult.getDossierId()) && typeRequest.getType() - .equals(typeResult.getType()) && typeRequest.getDossierTemplateId() - .equals(typeResult.getDossierTemplateId()) || typeRequest.getDossierId() == null && typeRequest.getType() - .equals(typeResult.getType()) && typeRequest.getDossierTemplateId() - .equals(typeResult.getDossierTemplateId()))) { - throw new ConflictException("The type already exists, could not be added again."); - } - String color = typeRequest.getHexColor(); - validateColor(color); - - String skippedHexColor = typeRequest.getSkippedHexColor(); - if (StringUtils.isBlank(skippedHexColor)) { //use the default value - skippedHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getSkippedColor(); - } else { - validateColor(typeRequest.getSkippedHexColor()); - } - - String recommendationHexColor = typeRequest.getRecommendationHexColor(); - if (StringUtils.isBlank(recommendationHexColor)) { //use the default value - recommendationHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getRecommendationColor(); - } else { - validateColor(typeRequest.getRecommendationHexColor()); - } - - return convert(dictionaryPersistenceService.addType(typeRequest.getType(), typeRequest.getDossierTemplateId(), color, recommendationHexColor, skippedHexColor, typeRequest.getRank(), typeRequest.isHint(), typeRequest.isCaseInsensitive(), typeRequest.isRecommendation(), typeRequest.getDescription(), typeRequest.isAddToDictionaryAction(), typeRequest.getLabel(), typeRequest.getDossierId(), typeRequest.isHasDictionary(), typeRequest.isSystemManaged(), typeRequest.isAutoHideSkipped()), Type.class); + return dictionaryService.addType(typeRequest); } @Override public void deleteType(@PathVariable(TYPE_PARAMETER_NAME) String typeId) { - // NotFoundException would be thrown if the type not found in database. - Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); - if (typeResult.isSystemManaged()) { - throw new BadRequestException("Can not delete system managed entity type"); - } - - var currentVersion = getCurrentVersion(typeResult); - - dictionaryPersistenceService.deleteType(typeId); - - entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY); - entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); - entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); - - dictionaryPersistenceService.incrementVersion(typeId); + dictionaryService.deleteType(typeId); } @@ -266,35 +109,6 @@ public class DictionaryController implements DictionaryResource { return convert(entries, DictionaryEntry.class); } - private Set getInvalidEntries(Set entries) { - - Predicate isDictionaryEntryNotValid = entry -> DictionaryValidator.validateDictionaryEntry(entry).isPresent(); - Predicate isStopword = stopwordService::isStopword; - - return entries.stream() - .filter(isDictionaryEntryNotValid.or(isStopword)) - .collect(toSet()); - } - - - private void validateBoolean(Boolean bool, String name) { - - Optional errorMessage = DictionaryValidator.validateBoolean(bool, name); - if (errorMessage.isPresent()) { - throw new BadRequestException(errorMessage.get()); - } - } - - - private void validateColor(String hexColor) { - - Optional errorMessage = DictionaryValidator.validateColor(hexColor); - if (errorMessage.isPresent()) { - throw new BadRequestException(errorMessage.get()); - } - - } - @Override public long getVersion(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId) { @@ -325,54 +139,4 @@ public class DictionaryController implements DictionaryResource { return convert(colorsService.getColors(dossierTemplateId), Colors.class); } - - private String cleanDictionaryEntry(String entry) { - - return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " "); - } - - - private void checkForDuplicateLabels(String dossierTemplateId, String dossierId, String type, String labelToCheck) { - - List typeResponse = dictionaryPersistenceService.getCumulatedTypes(dossierTemplateId, dossierId, false); - for (TypeEntity res : typeResponse) { - if (res.getDossierId() != null && res.getDossierId().equals(dossierId) && !type.equals(res.getType()) && res - .getDossierTemplateId() - .equals(dossierTemplateId) && labelToCheck.equals(res.getLabel()) || !type.equals(res.getType()) && res - .getDossierTemplateId() - .equals(dossierTemplateId) && labelToCheck.equals(res.getLabel())) { - throw new ConflictException("Label must be unique."); - } - } - } - - - private long getCurrentVersion(Type typeResult) { - - long currentVersion; - if (typeResult.getDossierId() != null) { - currentVersion = getVersionForDossier(typeResult.getDossierId()); - } else { - currentVersion = getVersion(typeResult.getDossierTemplateId()); - } - return currentVersion; - } - - - private String humanizedDictionaryType(String label) { - - String str = label; - str = str.replaceAll("-+?", " "); - str = str.replaceAll("_+?", " "); - str = str.replaceAll(" +", " "); - - StringBuilder strbf = new StringBuilder(); - Matcher match = Pattern.compile("([a-z])([a-z]*)", Pattern.CASE_INSENSITIVE).matcher(str); - while (match.find()) { - match.appendReplacement(strbf, match.group(1).toUpperCase() + match.group(2)); - } - return match.appendTail(strbf).toString(); - - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateController.java index 1ff4ea43f..a233c9a35 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateController.java @@ -1,11 +1,17 @@ package com.iqser.red.service.peristence.v1.server.controller; +import com.iqser.red.service.peristence.v1.server.model.DownloadJob; +import com.iqser.red.service.peristence.v1.server.service.DossierTemplateImportService; +import com.iqser.red.service.peristence.v1.server.service.export.DossierTemplateExportService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CreateOrUpdateDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportDownloadRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.resources.DossierTemplateResource; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -23,9 +29,12 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ma @RequiredArgsConstructor public class DossierTemplateController implements DossierTemplateResource { + private final DossierTemplateExportService dossierTemplateExportService; + private final DossierTemplateImportService dossierTemplateImportService; private final DossierTemplatePersistenceService dossierTemplatePersistenceService; private final DossierTemplateCloneService dossierTemplateCloneService; + @Override public DossierTemplate createOrUpdateDossierTemplate(@RequestBody CreateOrUpdateDossierTemplateRequest dossierTemplate) { return convert(dossierTemplatePersistenceService.createOrUpdateDossierTemplate(dossierTemplate), DossierTemplate.class); @@ -51,4 +60,19 @@ public class DossierTemplateController implements DossierTemplateResource { @RequestBody CloneDossierTemplateRequest cloneDossierTemplateRequest) { return convert(dossierTemplateCloneService.cloneDossierTemplate(dossierTemplateId,cloneDossierTemplateRequest), DossierTemplate.class); } + + @Override + public JSONPrimitive prepareExportDownload(@RequestBody ExportDownloadRequest request) { + + return dossierTemplateExportService.prepareExportDownload(request); + } + + public void createExportDownload(@RequestParam String userId, @RequestParam String storageId) { + dossierTemplateExportService.createDownloadArchive(DownloadJob.builder().userId(userId).storageId(storageId).build()); + } + + public void importDossierTemplate(@RequestBody ImportDossierTemplateRequest request) { + dossierTemplateImportService.importDossierTemplate(request); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReportTemplateController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReportTemplateController.java index a1d4f9429..4f1b482e4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReportTemplateController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReportTemplateController.java @@ -2,23 +2,21 @@ package com.iqser.red.service.peristence.v1.server.controller; import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.List; -import java.util.UUID; + +import com.iqser.red.service.peristence.v1.server.service.ReportTemplateService; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUpdateRequest; import org.apache.commons.io.IOUtils; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; -import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateDownload; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUpdateRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUploadRequest; import com.iqser.red.service.persistence.service.v1.api.resources.ReportTemplateResource; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; @@ -32,36 +30,12 @@ public class ReportTemplateController implements ReportTemplateResource { private final StorageService storageService; private final ReportTemplatePersistenceService reportTemplatePersistenceService; + private final ReportTemplateService reportTemplateService; public ReportTemplate uploadTemplate(@RequestBody ReportTemplateUploadRequest reportTemplateUploadRequest) { - String templateId = null; - List reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(reportTemplateUploadRequest.getDossierTemplateId()); - for (ReportTemplateEntity reportTemplate : reportTemplates) { - if (reportTemplate.getFileName() - .equals(reportTemplateUploadRequest.getFileName()) && reportTemplate.isMultiFileReport() && reportTemplateUploadRequest.isMultiFileReport() || reportTemplate.getFileName() - .equals(reportTemplateUploadRequest.getFileName()) && !reportTemplate.isMultiFileReport() && !reportTemplateUploadRequest.isMultiFileReport()) { - templateId = reportTemplate.getTemplateId(); - } - } - - String storageId = StorageIdUtils.getReportStorageId(reportTemplateUploadRequest.getDossierTemplateId(), reportTemplateUploadRequest.getFileName()); - storageService.storeObject(storageId, new ByteArrayInputStream(reportTemplateUploadRequest.getTemplate())); - - if (templateId != null) { - reportTemplatePersistenceService.updateTemplate(reportTemplateUploadRequest.getDossierTemplateId(), templateId, ReportTemplateUpdateRequest.builder() - .fileName(reportTemplateUploadRequest.getFileName()) - .multiFileReport(reportTemplateUploadRequest.isMultiFileReport()) - .activeByDefault(reportTemplateUploadRequest.isActiveByDefault()) - .build()); - } else { - templateId = UUID.randomUUID().toString(); - reportTemplatePersistenceService.insert(reportTemplateUploadRequest.getDossierTemplateId(), templateId, storageId, reportTemplateUploadRequest.getFileName(), - reportTemplateUploadRequest.isActiveByDefault(), reportTemplateUploadRequest.isMultiFileReport()); - } - - return convert(reportTemplatePersistenceService.find(templateId), ReportTemplate.class); + return reportTemplateService.uploadTemplate(reportTemplateUploadRequest); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java new file mode 100644 index 000000000..469747196 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java @@ -0,0 +1,284 @@ +package com.iqser.red.service.peristence.v1.server.service; + +import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities; +import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DictionaryService { + + private final EntryPersistenceService entryPersistenceService; + private final DictionaryPersistenceService dictionaryPersistenceService; + private final ColorsService colorsService; + private final StopwordService stopwordService; + + public Type addType(Type typeRequest) { + + if (typeRequest.getDossierTemplateId() == null) { + throw new BadRequestException("Dossier template id does not exist."); + } + + if (typeRequest.getLabel() == null || typeRequest.getLabel().isEmpty()) { + String label = humanizedDictionaryType(typeRequest.getType()); + typeRequest.setLabel(label); + } + + checkForDuplicateLabels(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), typeRequest.getType(), typeRequest + .getLabel()); + + if (this.checkForExistingType(typeRequest)) { + throw new ConflictException("The type already exists, could not be added again."); + } + String color = typeRequest.getHexColor(); + validateColor(color); + + String skippedHexColor = typeRequest.getSkippedHexColor(); + if (StringUtils.isBlank(skippedHexColor)) { //use the default value + skippedHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getSkippedColor(); + } else { + validateColor(typeRequest.getSkippedHexColor()); + } + + String recommendationHexColor = typeRequest.getRecommendationHexColor(); + if (StringUtils.isBlank(recommendationHexColor)) { //use the default value + recommendationHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getRecommendationColor(); + } else { + validateColor(typeRequest.getRecommendationHexColor()); + } + + return convert(dictionaryPersistenceService.addType(typeRequest.getType(), typeRequest.getDossierTemplateId(), color, recommendationHexColor, skippedHexColor, typeRequest.getRank(), typeRequest.isHint(), typeRequest.isCaseInsensitive(), typeRequest.isRecommendation(), typeRequest.getDescription(), typeRequest.isAddToDictionaryAction(), typeRequest.getLabel(), typeRequest.getDossierId(), typeRequest.isHasDictionary(), typeRequest.isSystemManaged(), typeRequest.isAutoHideSkipped()), Type.class); + } + + public void updateTypeValue(String typeId, Type typeRequest) { + + validateColor(typeRequest.getHexColor()); + validateBoolean(typeRequest.isHint(), "isHint"); + validateBoolean(typeRequest.isCaseInsensitive(), "isCaseInsensitive"); + String skippedHexColor = typeRequest.getSkippedHexColor(); + + if (StringUtils.isBlank(skippedHexColor)) { //use the default value + skippedHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getSkippedColor(); + typeRequest.setSkippedHexColor(skippedHexColor); + } else { + validateColor(skippedHexColor); + } + + String recommendationHexColor = typeRequest.getRecommendationHexColor(); + if (StringUtils.isBlank(recommendationHexColor)) { //use the default value + recommendationHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getRecommendationColor(); + typeRequest.setRecommendationHexColor(recommendationHexColor); + } else { + validateColor(typeRequest.getRecommendationHexColor()); + } + + + // To check whether the type exists + Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); + + if (typeRequest.getLabel() != null) { + checkForDuplicateLabels(typeResult.getDossierTemplateId(), typeResult.getDossierId(), typeResult.getType(), typeRequest.getLabel()); + } else { + typeRequest.setLabel(typeResult.getLabel()); + } + + dictionaryPersistenceService.updateType(typeId, convert(typeRequest, TypeEntity.class)); + + if (typeResult.isHint() != typeRequest.isHint() || typeResult.isCaseInsensitive() != typeRequest.isCaseInsensitive() || typeResult + .getRank() != typeRequest.getRank()) { + + var currentVersion = getCurrentVersion(typeResult); + + entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.ENTRY); + entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); + entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); + + } + + } + + public void addEntries(String typeId, List entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType) { + + Set cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet()); + + if (CollectionUtils.isEmpty(entries)) { + throw new BadRequestException("Entry list is empty."); + } + + var invalidEntries = getInvalidEntries(cleanEntries); + + if (!ignoreInvalidEntries && CollectionUtils.isNotEmpty(invalidEntries)) { + throw new BadRequestException("Error(s) validating dictionary entries:\\n" + String.join("\\n", invalidEntries)); + } else { + cleanEntries.removeAll(invalidEntries); + } + + // To check whether the type exists, type should not be added into database implicitly by addEntry. + Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); + if (!typeResult.isHasDictionary()) { + throw new BadRequestException("Entity type does not have a dictionary"); + } + + var currentVersion = getCurrentVersion(typeResult); + + if (removeCurrent) { + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType); + entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType); + } else { + entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType); + } + + dictionaryPersistenceService.incrementVersion(typeId); + } + + public void deleteEntries(String typeId, List entries, DictionaryEntryType dictionaryEntryType) { + + // To check whether the type exists + Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); + + var currentVersion = getCurrentVersion(typeResult); + + if (typeResult.isCaseInsensitive()) { + List existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY, null) + .stream() + .map(BaseDictionaryEntry::getValue) + .collect(toList()); + entryPersistenceService.deleteEntries(typeId, existing.stream() + .filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)) + .collect(toList()), currentVersion + 1, dictionaryEntryType); + } else { + entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType); + } + + dictionaryPersistenceService.incrementVersion(typeId); + } + + public void deleteType( String typeId) { + + // NotFoundException would be thrown if the type not found in database. + Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); + if (typeResult.isSystemManaged()) { + throw new BadRequestException("Can not delete system managed entity type"); + } + + var currentVersion = getCurrentVersion(typeResult); + + dictionaryPersistenceService.deleteType(typeId); + + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY); + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); + + dictionaryPersistenceService.incrementVersion(typeId); + } + + private void validateBoolean(Boolean bool, String name) { + + Optional errorMessage = DictionaryValidator.validateBoolean(bool, name); + if (errorMessage.isPresent()) { + throw new BadRequestException(errorMessage.get()); + } + } + + private void validateColor(String hexColor) { + + Optional errorMessage = DictionaryValidator.validateColor(hexColor); + if (errorMessage.isPresent()) { + throw new BadRequestException(errorMessage.get()); + } + + } + + private void checkForDuplicateLabels(String dossierTemplateId, String dossierId, String type, String labelToCheck) { + + List typeResponse = dictionaryPersistenceService.getCumulatedTypes(dossierTemplateId, dossierId, false); + for (TypeEntity res : typeResponse) { + if (res.getDossierId() != null && res.getDossierId().equals(dossierId) && !type.equals(res.getType()) && res + .getDossierTemplateId() + .equals(dossierTemplateId) && labelToCheck.equals(res.getLabel()) || !type.equals(res.getType()) && res + .getDossierTemplateId() + .equals(dossierTemplateId) && labelToCheck.equals(res.getLabel())) { + throw new ConflictException("Label must be unique."); + } + } + } + + private String humanizedDictionaryType(String label) { + + String str = label; + str = str.replaceAll("-+?", " "); + str = str.replaceAll("_+?", " "); + str = str.replaceAll(" +", " "); + + StringBuilder strbf = new StringBuilder(); + Matcher match = Pattern.compile("([a-z])([a-z]*)", Pattern.CASE_INSENSITIVE).matcher(str); + while (match.find()) { + match.appendReplacement(strbf, match.group(1).toUpperCase() + match.group(2)); + } + return match.appendTail(strbf).toString(); + + } + + private Set getInvalidEntries(Set entries) { + + Predicate isDictionaryEntryNotValid = entry -> DictionaryValidator.validateDictionaryEntry(entry).isPresent(); + Predicate isStopword = stopwordService::isStopword; + + return entries.stream() + .filter(isDictionaryEntryNotValid.or(isStopword)) + .collect(toSet()); + } + + private String cleanDictionaryEntry(String entry) { + + return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " "); + } + + private long getCurrentVersion(Type typeResult) { + + long currentVersion; + if (typeResult.getDossierId() != null) { + currentVersion = dictionaryPersistenceService.getVersionForDossier(typeResult.getDossierId()); + } else { + currentVersion = dictionaryPersistenceService.getVersion(typeResult.getDossierTemplateId()); + } + return currentVersion; + } + + public boolean checkForExistingType(Type typeRequest) { + return dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), false) + .stream() + .anyMatch(typeResult -> typeRequest.getDossierId() != null && typeResult.getDossierId() != null + && typeRequest.getDossierId().equals(typeResult.getDossierId()) + && typeRequest.getType().equals(typeResult.getType()) + && typeRequest.getDossierTemplateId().equals(typeResult.getDossierTemplateId()) + || typeRequest.getDossierId() == null && typeRequest.getType().equals(typeResult.getType()) + && typeRequest.getDossierTemplateId().equals(typeResult.getDossierTemplateId())); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java new file mode 100644 index 000000000..781fd43b5 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java @@ -0,0 +1,410 @@ +package com.iqser.red.service.peristence.v1.server.service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.*; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Watermark; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatusInfo; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportFilename; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportDossierTemplateRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportTemplateResult; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.RequestBody; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DossierTemplateImportService { + + private static final int THRESHOLD_ENTRIES = 10000; + private static final int THRESHOLD_SIZE = 1000000000; // 1 GB + private static final double THRESHOLD_RATIO = 10; + + private final DossierTemplateRepository dossierTemplateRepository; + private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + private final RulesPersistenceService rulesPersistenceService; + private final DossierTemplatePersistenceService dossierTemplatePersistenceService; + private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService; + private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; + private final ColorsService colorsService; + private final DossierStatusPersistenceService dossierStatusPersistenceService; + private final WatermarkService watermarkService; + private final ReportTemplateService reportTemplateService; + private final DictionaryService dictionaryService; + private final ObjectMapper objectMapper = new ObjectMapper(); + + public void importDossierTemplate(@RequestBody ImportDossierTemplateRequest request) { + ImportTemplateResult result = this.handleArchive(request); + if (result == null) { + throw new BadRequestException("Error while reading the archive"); + } + importDossierTemplate(result); + } + + public void importDossierTemplate(ImportTemplateResult request) { + String dossierTemplateId; + var dossierTemplateMeta = request.getDossierTemplate(); + var dossierTemplate = dossierTemplateRepository.findByIdAndNotDeleted(request.getDossierTemplateId()); + if (dossierTemplate.isPresent() && request.isUpdateExistingTemplate()) { + dossierTemplateId = dossierTemplate.get().getId(); + // override the existing dossier template + updateDossierTemplateMeta(dossierTemplate.get(), dossierTemplateMeta, request.getUserId()); + // set rules + rulesPersistenceService.setRules(request.getRuleSet(), dossierTemplateId); + + dossierTemplate.get().setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(dossierTemplate.get()).name())); + + } else { + // creates new dossier template + + if (StringUtils.isEmpty(dossierTemplateMeta.getName())) { + throw new ConflictException("DossierTemplate name must be set"); + } + dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName()); + DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity(); + dossierTemplateEntity.setId(UUID.randomUUID().toString()); + // order is important + BeanUtils.copyProperties(dossierTemplateMeta, dossierTemplateEntity); + dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + dossierTemplateEntity.setCreatedBy(request.getUserId()); + //set rules + rulesPersistenceService.setRules(request.getRuleSet(), dossierTemplateEntity.getId()); + var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity); + loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate)); + dossierTemplateId = loadedDossierTemplate.getId(); + + } + // set legal basis + if (CollectionUtils.isNotEmpty(request.getLegalBases())) { + legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases()); + } + //set dossier attributes + if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) { + dossierAttributeConfigPersistenceService.setDossierAttributesConfig(dossierTemplateId, + convert(request.getDossierAttributesConfigs(), DossierAttributeConfigEntity.class)); + } + // dossier status + if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) { + updateDossierStates(dossierTemplateId, request.getDossierStatusInfos()); + } + + //set file attributes + if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) { + fileAttributeConfigPersistenceService.setFileAttributesConfig(dossierTemplateId, + convert(request.getFileAttributesConfigs(), FileAttributeConfigEntity.class)); + } + if(request.getFileAttributesGeneralConfiguration() != null) { + fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId, + convert(request.getFileAttributesGeneralConfiguration(), FileAttributesGeneralConfigurationEntity.class)); + } + + //set report templates + if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) { + request.getReportTemplateUploadRequests().forEach(reportRequest -> { + reportRequest.setDossierTemplateId(dossierTemplateId); + reportTemplateService.uploadTemplate(reportRequest); + }); + } + + // set colors + if (request.getColors() != null) { + ColorsEntity colorsEntity = convert(request.getColors(), ColorsEntity.class); + colorsEntity.setDossierTemplateId(dossierTemplateId); + colorsService.saveColors(colorsEntity); + } + + // set watermarks + if (CollectionUtils.isNotEmpty(request.getWatermarks())) { + request.getWatermarks().forEach(watermark -> { + try { + watermarkService.createOrUpdateWatermark(watermark); + } catch (BadRequestException e) { + log.debug(" Conflict while validating watermark: " + watermark.getId()); + } + }); + } + + // set the types + // should delete existing types? in case of conflicts + for (var type: request.getTypes()) { + type.setDossierTemplateId(dossierTemplateId); + String typeId = type.getTypeId(); + if (dictionaryService.checkForExistingType(type)) { + dictionaryService.updateTypeValue(type.getTypeId(), type); + } else { + var returnedType = dictionaryService.addType(type); + typeId = returnedType.getTypeId(); + } + + if (request.getEntries() != null && !request.getEntries().isEmpty() + && !request.getEntries().get(type.getType()).isEmpty()) { + dictionaryService.addEntries(typeId, request.getEntries().get(type.getType()), true, true, DictionaryEntryType.ENTRY); + } + if (request.getFalsePositives() != null && !request.getFalsePositives().isEmpty() + && !request.getFalsePositives().get(type.getType()).isEmpty()) { + dictionaryService.addEntries(typeId, request.getFalsePositives().get(type.getType()), true, true, DictionaryEntryType.FALSE_POSITIVE); + } + if (request.getFalseRecommendations() != null && !request.getFalseRecommendations().isEmpty() + && !request.getFalseRecommendations().get(type.getType()).isEmpty()) { + dictionaryService.addEntries(typeId, request.getFalseRecommendations().get(type.getType()), true, true, DictionaryEntryType.FALSE_RECOMMENDATION); + } + } + + } + + private void updateDossierTemplateMeta(DossierTemplateEntity dossierTemplateEntity, DossierTemplate dossierTemplate, String userId) { + if (!dossierTemplateEntity.getName().equalsIgnoreCase(dossierTemplate.getName())) { + dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplate.getName()); + } + OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + dossierTemplateEntity.setName(dossierTemplate.getName()); + dossierTemplateEntity.setDescription(dossierTemplate.getDescription()); + dossierTemplateEntity.setDateModified(now); + dossierTemplateEntity.setModifiedBy(userId); + dossierTemplateEntity.setValidFrom(dossierTemplate.getValidFrom() == null ? dossierTemplateEntity.getValidFrom() : dossierTemplate.getValidFrom()); + dossierTemplateEntity.setValidTo(dossierTemplate.getValidTo() == null ? dossierTemplateEntity.getValidTo() : dossierTemplate.getValidTo()); + + dossierTemplateEntity.setDownloadFileTypes(dossierTemplate.getDownloadFileTypes() == null || + dossierTemplate.getDownloadFileTypes().isEmpty() ? dossierTemplateEntity.getDownloadFileTypes() : dossierTemplate.getDownloadFileTypes()); + + } + + private void updateDossierStates(String dossierTemplateId, List dossierStatusInfoList) { + dossierStatusInfoList.forEach(state -> { + var dossierStatusRequest = CreateOrUpdateDossierStatusRequest.builder() + .dossierStatusId(state.getId()) + .name(state.getName()) + .description(state.getDescription()) + .color(state.getColor()) + .dossierTemplateId(dossierTemplateId) + .rank(state.getRank()) + .build(); + try { + dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest); + } catch (BadRequestException e) { + log.debug(" Empty name for dossier status id " + state.getId()); + }catch (ConflictException e) { + log.debug(" Conflict detected for dossier status id: " + state.getId()); + } catch (NotFoundException e) { + log.debug(" Not found dossier status id " + state.getId()); + } + }); + } + + public ImportTemplateResult handleArchive(ImportDossierTemplateRequest request) { + + objectMapper.registerModule(new JavaTimeModule()); + File tempFile; + try { + FileAttribute> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); + tempFile = Files.createTempFile(UUID.randomUUID().toString(), ".zip", attr).toFile(); + IOUtils.write(request.getArchive(), new FileOutputStream(tempFile)); + var importTemplateResult = new ImportTemplateResult(); + importTemplateResult.setDossierTemplateId(request.getDossierTemplateId()); + importTemplateResult.setUserId(request.getUserId()); + importTemplateResult.setUpdateExistingTemplate(request.isUpdateExistingDossierTemplate()); + + try (FileInputStream fis = new FileInputStream(tempFile); BufferedInputStream bis = new BufferedInputStream(fis); + ZipArchiveInputStream zis = new ZipArchiveInputStream(bis)) { + + int totalSizeArchive = 0; + int totalEntryArchive = 0; + ZipArchiveEntry ze; + Map reportTemplateMap = new HashMap<>(); + Map reportTemplateBytesMap = new HashMap<>(); + List reportTemplateFilenameList = new ArrayList<>(); + Map> typeEntriesMap = new HashMap<>(); + Map> typeFalsePositivesMap = new HashMap<>(); + Map> typeFalseRecommendationsMap = new HashMap<>(); + + while ((ze = zis.getNextZipEntry()) != null) { + log.debug("---> " + ze.getName() + " ---- " + ze.isDirectory()); + totalEntryArchive++; + if (ze.isUnixSymlink()) { + continue; + } + if (!ze.isDirectory()) { + var bos = new ByteArrayOutputStream(); + var buffer = new byte[2048]; + var nBytes = 0; + int totalSizeEntry = 0; + + while ((nBytes = zis.read(buffer)) > 0) { + bos.write(buffer, 0, nBytes); + totalSizeEntry += nBytes; + totalSizeArchive += nBytes; + + double compressionRatio = (float) totalSizeEntry / ze.getCompressedSize(); + if (compressionRatio > THRESHOLD_RATIO) { + // ratio between compressed and uncompressed data is highly suspicious, looks like a Zip Bomb Attack + throw new BadRequestException("ZIP-Bomb detected."); + } + } + if (totalSizeArchive > THRESHOLD_SIZE) { + // the uncompressed data size is too much for the application resource capacity + throw new BadRequestException("ZIP-Bomb detected."); + } + + if (totalEntryArchive > THRESHOLD_ENTRIES) { + // too much entries in this archive, can lead to inodes exhaustion of the system + throw new BadRequestException("ZIP-Bomb detected."); + } + + var bytes = bos.toByteArray(); + if (ze.getName().contains(ExportFilename.DOSSIER_TEMPLATE_META.getFilename())) { + DossierTemplate dossierTemplate = objectMapper.readValue(bytes, DossierTemplate.class); + importTemplateResult.setDossierTemplate(dossierTemplate); + } else if (ze.getName().contains(ExportFilename.WATERMARK.getFilename())) { + List watermarkList = objectMapper.readValue(bytes, new TypeReference>() { + }); + importTemplateResult.getWatermarks().addAll(watermarkList); + } else if (ze.getName().contains(ExportFilename.COLORS.getFilename())) { + Colors colors = objectMapper.readValue(bytes, Colors.class); + importTemplateResult.setColors(colors); + } else if (ze.getName().contains(ExportFilename.DOSSIER_STATUS.getFilename())) { + List dossierStatusInfoList = objectMapper.readValue(bytes, new TypeReference>() { + }); + importTemplateResult.getDossierStatusInfos().addAll(dossierStatusInfoList); + } else if (ze.getName().contains(ExportFilename.DOSSIER_ATTRIBUTES_CONFIG.getFilename())) { + List dossierAttributeConfigs = objectMapper.readValue(bytes, new TypeReference>() { + }); + importTemplateResult.getDossierAttributesConfigs().addAll(dossierAttributeConfigs); + } else if (ze.getName().contains(ExportFilename.FILE_ATTRIBUTE_CONFIG.getFilename())) { + List fileAttributeConfigs = objectMapper.readValue(bytes, new TypeReference>() { + }); + importTemplateResult.getFileAttributesConfigs().addAll(fileAttributeConfigs); + } else if (ze.getName().contains(ExportFilename.LEGAL_BASIS.getFilename())) { + List legalBasisList = objectMapper.readValue(bytes, new TypeReference>() { + }); + importTemplateResult.getLegalBases().addAll(legalBasisList); + } else if (ze.getName().contains(ExportFilename.FILE_ATTRIBUTE_GENERAL_CONFIG.getFilename())) { + FileAttributesGeneralConfiguration fileAttributesGeneralConfiguration = objectMapper.readValue(bytes, FileAttributesGeneralConfiguration.class); + importTemplateResult.setFileAttributesGeneralConfiguration(fileAttributesGeneralConfiguration); + } else if (ze.getName().contains(ExportFilename.RULES.getFilename())) { + String rules = objectMapper.readValue(bytes, String.class); + importTemplateResult.setRuleSet(rules); + } else if (ze.getName().contains(ExportFilename.DOSSIER_TYPE.getFilename())) { + Type type = objectMapper.readValue(bytes, Type.class); + importTemplateResult.getTypes().add(type); + } else if (ze.getName().contains(ExportFilename.ENTRIES.getFilename())) { + + List entries = this.readEntries(bytes); + typeEntriesMap.put(this.getTypeName(ze.getName()), entries); + importTemplateResult.setEntries(typeEntriesMap); + } else if (ze.getName().contains(ExportFilename.FALSE_POSITIVES.getFilename())) { + + List falsePositives = this.readEntries(bytes); + typeFalsePositivesMap.put(this.getTypeName(ze.getName()), falsePositives); + importTemplateResult.setFalsePositives(typeFalsePositivesMap); + } else if (ze.getName().contains(ExportFilename.FALSE_RECOMMENDATION.getFilename())) { + + List falseRecommendations = this.readEntries(bytes); + typeFalseRecommendationsMap.put(this.getTypeName(ze.getName()), falseRecommendations); + importTemplateResult.setFalseRecommendations(typeFalseRecommendationsMap); + } else if (ze.getName().contains("reportTemplateList.json")) { + var reportTemplateList = objectMapper.readValue(bytes, new TypeReference>() { + }); + reportTemplateMap = reportTemplateList.stream() + .collect(Collectors.toMap(ReportTemplate::getFileName, Function.identity())); + reportTemplateFilenameList = reportTemplateList.stream().map(rt -> rt.getFileName()).collect(Collectors.toList()); + } else { + reportTemplateBytesMap.put(ze.getName(), bos); + } + bos.close(); + } + } + List reportTemplateUploadRequests = new ArrayList<>(); + + for (var reportZe : reportTemplateBytesMap.entrySet()) { + if (reportTemplateFilenameList.contains(reportZe.getKey())) { + var report = reportTemplateMap.get(reportZe.getKey()); + importTemplateResult.getReportTemplateUploadRequests().add(ReportTemplateUploadRequest.builder() + .fileName(report.getFileName()) + .activeByDefault(report.isActiveByDefault()) + .multiFileReport(report.isMultiFileReport()) + .dossierTemplateId(request.getDossierTemplateId()) + .template(reportZe.getValue().toByteArray()) + .build()); + + } + importTemplateResult.getReportTemplateUploadRequests().addAll(reportTemplateUploadRequests); + } + return importTemplateResult; + } finally { + if (tempFile != null) { + boolean isDeleted = tempFile.delete(); + if (!isDeleted) { + log.debug("tempFile could not be deleted"); + } + } + } + } catch (IOException e) { + log.debug("exception: ", e); + } catch (BadRequestException e) { + log.debug("exception: ", e); + } + return null; + } + + private List readEntries(byte[] bytes ) { + ByteArrayInputStream bInput = new ByteArrayInputStream(bytes); + DataInputStream in = new DataInputStream(bInput); + List entries = new ArrayList<>(); + try { + while (in.available() > 0) { + String entry = in.readUTF(); + entries.add(entry); + + } + } catch (IOException e) { + log.debug("exception: ", e); + } + return entries; + } + + private String getTypeName(String filename) { + var index = filename.indexOf('/'); + return filename.substring(0, index); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReportTemplateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReportTemplateService.java new file mode 100644 index 000000000..27e873deb --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReportTemplateService.java @@ -0,0 +1,61 @@ +package com.iqser.red.service.peristence.v1.server.service; + +import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUploadRequest; +import com.iqser.red.storage.commons.service.StorageService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.util.List; +import java.util.UUID; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ReportTemplateService { + + private final StorageService storageService; + private final ReportTemplatePersistenceService reportTemplatePersistenceService; + + public ReportTemplate uploadTemplate(ReportTemplateUploadRequest reportTemplateUploadRequest) { + String templateId = null; + List reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(reportTemplateUploadRequest + .getDossierTemplateId()); + for (ReportTemplateEntity reportTemplate : reportTemplates) { + if (reportTemplate.getFileName() + .equals(reportTemplateUploadRequest.getFileName()) && reportTemplate.isMultiFileReport() && reportTemplateUploadRequest + .isMultiFileReport() || reportTemplate.getFileName() + .equals(reportTemplateUploadRequest.getFileName()) && !reportTemplate.isMultiFileReport() && !reportTemplateUploadRequest + .isMultiFileReport()) { + templateId = reportTemplate.getTemplateId(); + } + } + + String storageId = StorageIdUtils.getReportStorageId(reportTemplateUploadRequest.getDossierTemplateId(), reportTemplateUploadRequest + .getFileName()); + storageService.storeObject(storageId, new ByteArrayInputStream(reportTemplateUploadRequest.getTemplate())); + + if (templateId != null) { + reportTemplatePersistenceService.updateTemplate(reportTemplateUploadRequest.getDossierTemplateId(), templateId, ReportTemplateUpdateRequest + .builder() + .fileName(reportTemplateUploadRequest.getFileName()) + .multiFileReport(reportTemplateUploadRequest.isMultiFileReport()) + .activeByDefault(reportTemplateUploadRequest.isActiveByDefault()) + .build()); + } else { + templateId = UUID.randomUUID().toString(); + reportTemplatePersistenceService.insert(reportTemplateUploadRequest.getDossierTemplateId(), templateId, storageId, reportTemplateUploadRequest + .getFileName(), reportTemplateUploadRequest.isActiveByDefault(), reportTemplateUploadRequest.isMultiFileReport()); + } + + return convert(reportTemplatePersistenceService.find(templateId), ReportTemplate.class); + } +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java new file mode 100644 index 000000000..8a8886b69 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java @@ -0,0 +1,207 @@ +package com.iqser.red.service.peristence.v1.server.service.export; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.peristence.v1.server.model.DownloadJob; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.peristence.v1.server.utils.FileSystemBackedArchiver; +import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; +import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.*; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Watermark; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatusInfo; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportDownloadRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportFilename; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import java.io.*; +import java.util.List; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DossierTemplateExportService { + + private static final String JSON_EXT = ".json"; + private static final String TXT_EXT = ".txt"; + private final DossierTemplatePersistenceService dossierTemplatePersistenceService; + private final DownloadStatusPersistenceService downloadStatusPersistenceService; + private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService; + private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; + private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + private final RulesPersistenceService rulesPersistenceService; + private final FileManagementStorageService fileManagementStorageService; + private final ColorsService colorsService; + private final EntryPersistenceService entryPersistenceService; + private final ObjectMapper objectMapper = new ObjectMapper(); + + private final RabbitTemplate rabbitTemplate; + + public JSONPrimitive prepareExportDownload(ExportDownloadRequest request) { + + var mimeType = "application/zip"; + + String downloadFilename = request.getDossierTemplateId() + ".zip"; + String storageId = StorageIdUtils.getStorageId(request.getUserId(), request.getDossierTemplateId()); + + downloadStatusPersistenceService.createStatus(request.getUserId(), storageId, downloadFilename, mimeType); + addToExportDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).build(), 1); + + return new JSONPrimitive<>(storageId); + } + + @SneakyThrows + public void createDownloadArchive(DownloadJob downloadJob) { + objectMapper.registerModule(new JavaTimeModule()); + DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId()); + + String dossierTemplateId = extractDossierTemplateId(downloadStatus.getFilename()); + var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate((dossierTemplateId)); + + try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) { + // add dossier template name and meta data + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.DOSSIER_TEMPLATE_META, JSON_EXT), + objectMapper.writeValueAsBytes(convert(dossierTemplate, DossierTemplate.class)))); + + //add watermark json file + var watermarkList = dossierTemplate.getWatermarkConfigs(); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.WATERMARK, JSON_EXT), + objectMapper.writeValueAsBytes(convert(watermarkList, Watermark.class)))); + + //add colors json file + var colors = colorsService.getColors(dossierTemplateId); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.COLORS, JSON_EXT), + objectMapper.writeValueAsBytes(convert(colors, Colors.class)))); + + // add dossier statuses + var dossierStatusList = dossierTemplate.getDossierStatusList(); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.DOSSIER_STATUS, JSON_EXT), + objectMapper.writeValueAsBytes(convert(dossierStatusList, DossierStatusInfo.class)))); + + // add dossier attributes config json + var dossierAttributesConfig = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.DOSSIER_ATTRIBUTES_CONFIG, JSON_EXT), + objectMapper.writeValueAsBytes(convert(dossierAttributesConfig, DossierAttributeConfig.class)))); + + // add file attribute configs + var fileAttributeConfigList = dossierTemplate.getFileAttributeConfigs(); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.FILE_ATTRIBUTE_CONFIG, JSON_EXT), + objectMapper.writeValueAsBytes(convert(fileAttributeConfigList, FileAttributeConfig.class)))); + + // add legal basis mapping + var legalBasisMappingList = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.LEGAL_BASIS, JSON_EXT), + objectMapper.writeValueAsBytes(convert(legalBasisMappingList, LegalBasis.class)))); + + // add file attributes general configuration + var fileAttributesGeneralConfig = fileAttributeConfigPersistenceService.getFileAttributesGeneralConfiguration(dossierTemplateId); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.FILE_ATTRIBUTE_GENERAL_CONFIG, JSON_EXT), + objectMapper.writeValueAsBytes(convert(fileAttributesGeneralConfig, FileAttributesGeneralConfiguration.class)))); + + // add rule set + var ruleSet = rulesPersistenceService.getRules(dossierTemplateId); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.RULES, TXT_EXT), + objectMapper.writeValueAsBytes(ruleSet.getValue()))); + + //N files with the related report templates + var reportTemplateList = dossierTemplate.getReportTemplates(); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.REPORT_TEMPLATE, JSON_EXT), + objectMapper.writeValueAsBytes(convert(reportTemplateList, ReportTemplate.class)))); + reportTemplateList.forEach(reportTemplate -> { + var report = fileManagementStorageService.getStoredObjectBytes(reportTemplate.getStorageId()); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, reportTemplate.getFileName(), report)); + }); + // for every type 1 folder with: + // 1 json file containing meta info of the type + // and 1 txt file for every type: entries, false positives and false recommendation + var dossierTypes = dossierTemplate.getDossierTypes(); + for (TypeEntity typeEntity : dossierTypes) { + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getType(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT), + objectMapper.writeValueAsBytes(convert(typeEntity, Type.class)))); + + var entriesValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.ENTRY, null) + .stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList()); + var falsePositiveValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.FALSE_POSITIVE, null) + .stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList()); + var falseRecommendationValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.FALSE_RECOMMENDATION, null) + .stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList()); + + writeEntriesListToFile(fileSystemBackedArchiver, entriesValuesList, typeEntity.getType(), getFilename(ExportFilename.ENTRIES, TXT_EXT)); + writeEntriesListToFile(fileSystemBackedArchiver, falsePositiveValuesList, typeEntity.getType(),getFilename(ExportFilename.FALSE_POSITIVES, TXT_EXT)); + writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(),getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT)); + } + + storeZipFile(downloadStatus.getStorageId(), fileSystemBackedArchiver); + downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength()); + + } catch (JsonProcessingException e) { + log.debug("fail ", e); + throw new BadRequestException("Error creating the archive"); // update error handling + } + + } + + private String getFilename(ExportFilename exportFilename, String extension) { + return exportFilename.getFilename() + extension; + } + + private void writeEntriesListToFile(FileSystemBackedArchiver fileSystemBackedArchiver, List entriesValuesList, String folderName, String filename) { + try { + ByteArrayOutputStream bt = new ByteArrayOutputStream(); + DataOutputStream dt = new DataOutputStream(bt); + for(String entry : entriesValuesList) { + dt.writeUTF(entry); + } + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folderName, filename, bt.toByteArray())); + dt.close(); + bt.close(); + } catch (IOException e) { + log.debug("Error writing values to files"); + } + } + + private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) { + long start = System.currentTimeMillis(); + fileManagementStorageService.storeObject(storageId, fileSystemBackedArchiver.toInputStream()); + log.info("Successfully stored zip for downloadId {}, took {}", storageId, System.currentTimeMillis() - start); + } + + private String extractDossierTemplateId(String fileName) { + var lastIndexOfDot = fileName.lastIndexOf("."); + return fileName.substring(0, lastIndexOfDot); + } + + private void addToExportDownloadQueue(DownloadJob downloadJob, int priority) { + + try { + rabbitTemplate.convertAndSend(MessagingConfiguration.DOWNLOAD_QUEUE, objectMapper.writeValueAsString(downloadJob), message -> { + message.getMessageProperties().setPriority(priority); + return message; + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/ExportDownloadMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/ExportDownloadMessageReceiver.java new file mode 100644 index 000000000..c5b97a751 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/ExportDownloadMessageReceiver.java @@ -0,0 +1,30 @@ +package com.iqser.red.service.peristence.v1.server.service.export; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.peristence.v1.server.model.DownloadJob; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +@RabbitListener(queues = MessagingConfiguration.EXPORT_DOWNLOAD_QUEUE) +public class ExportDownloadMessageReceiver { + private final DossierTemplateExportService dossierTemplateService; + private final ObjectMapper objectMapper; + + @RabbitHandler + public void receive(String in) throws JsonProcessingException { + + DownloadJob downloadJob = objectMapper.readValue(in, DownloadJob.class); + log.info("Preparing export download for userId: {} and storageId: {}", downloadJob.getUserId(), downloadJob.getStorageId()); + + dossierTemplateService.createDownloadArchive(downloadJob); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/StorageIdUtils.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/StorageIdUtils.java index 0cc7ec3b6..ac1a34c38 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/StorageIdUtils.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/StorageIdUtils.java @@ -17,6 +17,11 @@ public final class StorageIdUtils { return userId + "/" + dossierId + "/" + filename; } + public static String getStorageId(String userId, String dossierTemplateId) { + + return userId + "/" + dossierTemplateId; + } + public static String getReportStorageId(String dossierTemplateId, String fileName) { return dossierTemplateId + "/" + fileName; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index 4ef28b492..c38361d4f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -1,9 +1,12 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; +import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.integration.client.*; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.peristence.v1.server.service.export.ExportDownloadMessageReceiver; +import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.*; @@ -14,18 +17,22 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttributeType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportDownloadRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; +import com.iqser.red.storage.commons.service.StorageService; import feign.FeignException; +import lombok.SneakyThrows; import org.junit.Test; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; -import java.util.Collections; -import java.util.List; +import java.util.*; import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; import static org.assertj.core.api.Assertions.assertThat; @@ -62,6 +69,30 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { @Autowired private DossierStatusClient dossierStatusClient; + @Autowired + private DownloadClient downloadClient; + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private ExportDownloadMessageReceiver exportDownloadReportMessageReceiver; + + @Autowired + private StorageService storageService; + + @Test + public void testDownload() { + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + // test the export of dossier template + dossierTemplateClient.prepareExportDownload(ExportDownloadRequest.builder() + .userId("1") + .dossierTemplateId(dossierTemplate.getId()) + .build()); + var statuses = downloadClient.getDownloadStatus("1"); + assertThat(statuses).isNotEmpty(); + } @Test public void testDossierTemplate() { @@ -115,6 +146,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { @Test + @SneakyThrows public void testCloneDossierTemplate() { var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); @@ -285,4 +317,190 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { .isEqualTo(watermarkClient.getWatermarksForDossierTemplateId(clonedDT.getId()).get(0).getName()); } + @SneakyThrows + @Test + public void testExportDossierTemplate() { +// TenantContext.setTenantId(DEFAULT_TENANT); + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + setupDossierTemplate(dossierTemplate); + + dossierTemplateClient.prepareExportDownload(ExportDownloadRequest.builder() + .userId("1") + .dossierTemplateId(dossierTemplate.getId()) + .build()); + var statuses = downloadClient.getDownloadStatus("1"); + assertThat(statuses).isNotEmpty(); + + +// var downloadJob = DownloadJob.builder() +// .storageId(StorageIdUtils.getStorageId("1", dossierTemplate.getId())) +// .userId("1").build(); +// exportDownloadReportMessageReceiver.receive(objectMapper.writeValueAsString(downloadJob)); + String storageId = StorageIdUtils.getStorageId("1", dossierTemplate.getId()); + dossierTemplateClient.createExportDownload("1", storageId); + + statuses = downloadClient.getDownloadStatus("1"); + assertThat(statuses).isNotEmpty(); + assertThat(statuses.get(0).getStatus()).isEqualTo(DownloadStatusValue.READY); + } + + @Test + @SneakyThrows + public void testImportDossierTemplateUpdateExisting() { + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + setupDossierTemplate(dossierTemplate); + // prepare an archive + dossierTemplateClient.prepareExportDownload(ExportDownloadRequest.builder() + .userId("1") + .dossierTemplateId(dossierTemplate.getId()) + .build()); + String storageId = StorageIdUtils.getStorageId("1", dossierTemplate.getId()); + dossierTemplateClient.createExportDownload("1", storageId); + + var statuses = downloadClient.getDownloadStatus("1"); + assertThat(statuses).isNotEmpty(); + assertThat(statuses.get(0).getStatus()).isEqualTo(DownloadStatusValue.READY); + + ImportDossierTemplateRequest request1 = ImportDossierTemplateRequest.builder() + .dossierTemplateId(dossierTemplate.getId()) + .updateExistingDossierTemplate(true) + .userId("1") + .archive(storageService.getObject(storageId).getInputStream().readAllBytes()) + .build(); + + dossierTemplateClient.importDossierTemplate(request1); + var updatedDossierTemplate = dossierTemplateClient.getDossierTemplate(dossierTemplate.getId()); + assertThat(updatedDossierTemplate.getModifiedBy()).isEqualTo("1"); + assertThat(updatedDossierTemplate.getDateModified()).isNotNull(); + assertThat(watermarkClient.getWatermarksForDossierTemplateId(updatedDossierTemplate.getId()).size()).isEqualTo(1); + + } + + private void setupDossierTemplate(DossierTemplate dossierTemplate) { + Type type = Type.builder() + .type("t") + .dossierTemplateId(dossierTemplate.getId()) + .hexColor("#ddddd") + .recommendationHexColor("#cccccc") + .skippedHexColor("#cccccc") + .rank(999) + .isHint(false) + .isRecommendation(false) + .isCaseInsensitive(false) + .description("t.getDescription()") + .addToDictionaryAction(true) + .label("t.getLabel()") + .dossierId(null) + .version(23) + .entries(List.of(DictionaryEntry.builder().entryId(1001).value("dictEntry1").version(23).deleted(false).typeId("t.getType()").build())) + .falsePositiveEntries(List.of(DictionaryEntry.builder().entryId(2001).value("dictEntry2").version(23).deleted(false).typeId("t.getType()").build())) + .falseRecommendationEntries(List.of(DictionaryEntry.builder().entryId(3001).value("dictEntry3").version(23).deleted(false).typeId("t.getType()").build())) + .hasDictionary(true) + .systemManaged(false) + .build(); + Type type2 = Type.builder() + .type("t2") + .dossierTemplateId(dossierTemplate.getId()) + .hexColor("#12345") + .recommendationHexColor("#6789a") + .skippedHexColor("#6789a") + .rank(1002) + .isHint(false) + .isRecommendation(false) + .isCaseInsensitive(false) + .description("t2.getDescription()") + .addToDictionaryAction(true) + .label("t2.getLabel()") + .dossierId(null) + .version(23) + .entries(List.of(DictionaryEntry.builder().entryId(1001).value("dictEntry1").version(23).deleted(false).typeId("t2.getType()").build())) + .falsePositiveEntries(List.of(DictionaryEntry.builder().entryId(2001).value("dictEntry2").version(23).deleted(false).typeId("t2.getType()").build())) + .falseRecommendationEntries(List.of(DictionaryEntry.builder().entryId(3001).value("dictEntry3").version(23).deleted(false).typeId("t2.getType()").build())) + .hasDictionary(true) + .systemManaged(false) + .build(); + var createdType1 = dictionaryClient.addType(type); + var createdType2 = dictionaryClient.addType(type2); + var loadedType1 = dictionaryClient.getDictionaryForType(createdType1.getId(), null); + var loadedType2 = dictionaryClient.getDictionaryForType(createdType2.getId(), null); + dictionaryClient.addEntries(loadedType1.getTypeId(), List.of("entry1", "entry2"), false, false, DictionaryEntryType.ENTRY); + dictionaryClient.addEntries(loadedType2.getTypeId(), List.of("entry3", "entry4"), false, false, DictionaryEntryType.FALSE_POSITIVE); + dossierAttributeConfigClient.setDossierAttributesConfig(dossierTemplate.getId(), List.of(DossierAttributeConfig.builder() + .dossierTemplateId(dossierTemplate.getId()) + .editable(false) + .id("dossierAttributeId") + .label("labelDossierAttribute") + .type(DossierAttributeType.TEXT) + .placeholder("placeholderDossier") + .build())); + fileAttributeConfigClient.setFileAttributesGeneralConfig(dossierTemplate.getId(), FileAttributesGeneralConfiguration.builder().dossierTemplateId(dossierTemplate.getId()) + .delimiter("") + .filenameMappingColumnHeaderName("filenameMappingColumnHeaderName") + .build()); + fileAttributeConfigClient.setFileAttributesConfig(dossierTemplate.getId(), List.of(FileAttributeConfig.builder() + .dossierTemplateId(dossierTemplate.getId()) + .primaryAttribute(true) + .csvColumnHeader("12345") + .displayedInFileList(false) + .editable(false) + .filterable(false) + .id("fileAttributeId") + .label("labelFileAttribute") + .type(FileAttributeType.TEXT) + .placeholder("placeholderFile") + .build())); + ReportTemplateEntity rte = ReportTemplateEntity.builder() + .templateId("templateId") + .dossierTemplate(convert(dossierTemplate, DossierTemplateEntity.class)) + .dossierTemplateId(dossierTemplate.getId()) + .dossiers(null) + .activeByDefault(false) + .fileName("rte") + .multiFileReport(false) + .storageId("storageId") + .uploadDate(OffsetDateTime.now()) + .build(); + reportTemplateClient.uploadTemplate(ReportTemplateUploadRequest.builder() + .template("some text".getBytes(StandardCharsets.UTF_8)) + .dossierTemplateId(dossierTemplate.getId()) + .fileName("Report Template") + .activeByDefault(true) + .multiFileReport(false) + .build()); + var col = Colors.builder() + .analysisColor("#111111") + .dossierTemplateId(dossierTemplate.getId()) + .dictionaryRequestColor("#333333") + .requestAddColor("#444444") + .ignoredHintColor("#555555") + .requestRemoveColor("#666666") + .analysisColor("#777777") + .previewColor("#888888") + .redactionColor("#999999") + .updatedColor("#aaaaaa") + .build(); + dictionaryClient.setColors(dossierTemplate.getId(), col); + var dossierStatus = CreateOrUpdateDossierStatusRequest.builder() + .name("dossStatus1") + .description("ds description") + .color("#115599") + .rank(456) + .dossierTemplateId(dossierTemplate.getId()) + .build(); + dossierStatusClient.createOrUpdateDossierStatus(dossierStatus); + + Watermark watermark = new Watermark(); + watermark.setName("watermark name"); + watermark.setEnabled(true); + watermark.setText("Minions ipsum chasy para tu la bodaaa bananaaaa hana dul sae. Chasy hana dul sae pepete hana dul sae belloo! Tatata bala tu ti aamoo! Jeje."); + watermark.setFontSize(12); + watermark.setFontType("font"); + watermark.setHexColor("#dddddd"); + watermark.setOpacity(20); + watermark.setOrientation(WatermarkOrientation.DIAGONAL); + watermark.setDossierTemplateId(dossierTemplate.getId()); + watermark.setCreatedBy("user"); + + watermarkClient.createOrUpdateWatermark(watermark); + } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java index 8d989ba07..a5ccdf2e5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java @@ -113,7 +113,9 @@ public class DossierTest extends AbstractPersistenceServerServiceTest { // disable the watermark used watermarkConfig.setEnabled(false); - watermarkClient.createOrUpdateWatermark(watermarkConfig); + var watermarkDisabled = watermarkClient.createOrUpdateWatermark(watermarkConfig); + assertThat(watermarkDisabled.isEnabled()).isFalse(); + assertThat(watermarkDisabled.getId()).isEqualTo(watermarkConfig.getId()); // update dossier description, while the watermark used was disabled cru.setDescription("new description"); updated = dossierClient.updateDossier(cru, dossier.getId());