Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-2200 - Dossier Template export and import

Browse Source 
- add implementation for import and export.
- add tests
This commit is contained in:
devplant 2022-09-14 09:42:06 +03:00
parent 87127b8295
commit 35f3b3e06c
19 changed files with 1440 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportDownloadRequest.java Normal file
View File

@ -0,0 +1,12 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport;
import lombok.*;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class ExportDownloadRequest {
private String userId;
private String dossierTemplateId;
}

29
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ExportFilename.java Normal file
View File

@ -0,0 +1,29 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport;
import lombok.Getter;
public enum ExportFilename {
DOSSIER_TEMPLATE_META("dossierTemplate"),
WATERMARK("watermark"),
COLORS("colors"),
DOSSIER_STATUS("dossierStatusList"),
DOSSIER_ATTRIBUTES_CONFIG("dossierAttributesConfigList"),
FILE_ATTRIBUTE_CONFIG("fileAttributeConfigList"),
FILE_ATTRIBUTE_GENERAL_CONFIG("fileAttributeGeneralConfigList"),
LEGAL_BASIS("legalBasisMappingList"),
RULES("rules"),
REPORT_TEMPLATE("reportTemplateList"),
DOSSIER_TYPE("dossierType"),
ENTRIES("entries"),
FALSE_POSITIVES("falsePositives"),
FALSE_RECOMMENDATION("falseRecommendations");
@Getter
private final String filename;
ExportFilename(String filename) {
this.filename = filename;
}
}

18
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportDossierTemplateRequest.java Normal file
View File

@ -0,0 +1,18 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport;
import lombok.*;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class ImportDossierTemplateRequest {
@NonNull
private byte[] archive;
private String dossierTemplateId;
private String userId;
private boolean updateExistingDossierTemplate;
}

72
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/importexport/ImportTemplateResult.java Normal file
View File

@ -0,0 +1,72 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.FileAttributesGeneralConfiguration;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUploadRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Watermark;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatusInfo;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ImportTemplateResult {
public String dossierTemplateId;
public String userId;
public boolean updateExistingTemplate;
public DossierTemplate dossierTemplate;
@Builder.Default
public List<Watermark> watermarks = new ArrayList<>();
public Colors colors;
@Builder.Default
public List<DossierStatusInfo> dossierStatusInfos = new ArrayList<>();
@Builder.Default
public List<DossierAttributeConfig> dossierAttributesConfigs = new ArrayList<>();
public FileAttributesGeneralConfiguration fileAttributesGeneralConfiguration;
@Builder.Default
public List<FileAttributeConfig> fileAttributesConfigs = new ArrayList<>();
@Builder.Default
public List<LegalBasis> legalBases = new ArrayList<>();
public String ruleSet;
@Builder.Default
public List<ReportTemplateUploadRequest> reportTemplateUploadRequests = new ArrayList<>();
@Builder.Default
public List<Type> types= new ArrayList<>();
@Builder.Default
public Map<String, List<String>> entries = new HashMap<>();
@Builder.Default
public Map<String, List<String>> falsePositives = new HashMap<>();
@Builder.Default
public Map<String, List<String>> falseRecommendations = new HashMap<>();
}

15
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateResource.java
View File

@ -1,8 +1,12 @@
package com.iqser.red.service.persistence.service.v1.api.resources;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CreateOrUpdateDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportDownloadRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportTemplateResult;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
@ -19,6 +23,8 @@ public interface DossierTemplateResource {
String USER_ID_PARAM = "userId";
String CLONE_PATH = "/clone";
String EXPORT_PATH = "/export";
String IMPORT_PATH = "/import";
@ResponseBody
@ResponseStatus(HttpStatus.ACCEPTED)
@ -46,4 +52,13 @@ public interface DossierTemplateResource {
DossierTemplate cloneDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId,
@RequestBody CloneDossierTemplateRequest cloneDossierTemplateRequest);
@PostMapping(value = DOSSIER_TEMPLATE_PATH + EXPORT_PATH + "/prepare", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
JSONPrimitive<String> prepareExportDownload(@RequestBody ExportDownloadRequest request);
@PostMapping(value = DOSSIER_TEMPLATE_PATH + EXPORT_PATH + "/create", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
void createExportDownload(@RequestParam String userId, @RequestParam String storageId);
@PostMapping(value = DOSSIER_TEMPLATE_PATH + IMPORT_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
void importDossierTemplate(@RequestBody ImportDossierTemplateRequest request);
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -38,12 +39,18 @@ public class DownloadStatusPersistenceService {
downloadStatus.setMimeType(mimeType);
downloadStatus.setDossier(dossier);
downloadStatus.setCreationDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
downloadStatus.setFiles(fileRepository.findAllById(fileIds));
downloadStatus.setDownloadFileTypes(new HashSet<>(downloadFileTypes));
downloadStatus.setFiles(fileIds!= null ? fileRepository.findAllById(fileIds) : new ArrayList<>());
downloadStatus.setDownloadFileTypes(downloadFileTypes != null ?new HashSet<>(downloadFileTypes) : new HashSet<>());
downloadStatusRepository.save(downloadStatus);
}
// use this to create a status for export dossier template.
public void createStatus(String userId, String storageId, String filename, String mimeType) {
this.createStatus(userId, storageId, null, filename, mimeType, null, null);
}
@Transactional
public void updateStatus(String storageId, DownloadStatusValue status) {

13
persistence-service-v1/persistence-service-server-v1/pom.xml
View File

@ -93,6 +93,19 @@
<groupId>com.iqser.red.commons</groupId>
<artifactId>spring-commons</artifactId>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>jackson-commons</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.21</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
<dependency>
<groupId>org.postgresql</groupId>

16
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java
View File

@ -18,6 +18,9 @@ public class MessagingConfiguration {
public static final String DOWNLOAD_QUEUE = "downloadQueue";
public static final String DOWNLOAD_DLQ = "downloadDLQ";
public static final String EXPORT_DOWNLOAD_QUEUE = "exportDownloadQueue";
public static final String EXPORT_DOWNLOAD_DLQ = "exportDownloadDLQ";
public static final String REPORT_QUEUE = "reportQueue";
public static final String REPORT_DLQ = "reportDLQ";
@ -159,6 +162,19 @@ public class MessagingConfiguration {
return QueueBuilder.durable(DOWNLOAD_DLQ).build();
}
@Bean
public Queue exportDownloadQueue() {
return QueueBuilder.durable(EXPORT_DOWNLOAD_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", EXPORT_DOWNLOAD_DLQ).build();
}
@Bean
public Queue exportDownloadDeadLetterQueue() {
return QueueBuilder.durable(EXPORT_DOWNLOAD_DLQ).build();
}
@Bean
public Queue reportQueue() {

250
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java
View File

@ -1,13 +1,7 @@
package com.iqser.red.service.peristence.v1.server.controller;
import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities;
import com.iqser.red.service.peristence.v1.server.service.StopwordService;
import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.peristence.v1.server.service.DictionaryService;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
@ -18,8 +12,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
@ -27,15 +19,8 @@ import org.springframework.web.bind.annotation.RestController;
import javax.transaction.Transactional;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
@RestController
@RequiredArgsConstructor
@ -45,7 +30,7 @@ public class DictionaryController implements DictionaryResource {
private final EntryPersistenceService entryPersistenceService;
private final DictionaryPersistenceService dictionaryPersistenceService;
private final ColorsService colorsService;
private final StopwordService stopwordService;
private final DictionaryService dictionaryService;
@Override
@ -54,177 +39,35 @@ public class DictionaryController implements DictionaryResource {
@RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries,
@RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) {
Set<String> cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet());
if (CollectionUtils.isEmpty(entries)) {
throw new BadRequestException("Entry list is empty.");
}
var invalidEntries = getInvalidEntries(cleanEntries);
if (!ignoreInvalidEntries && CollectionUtils.isNotEmpty(invalidEntries)) {
throw new BadRequestException("Error(s) validating dictionary entries:\\n" + String.join("\\n", invalidEntries));
} else {
cleanEntries.removeAll(invalidEntries);
}
// To check whether the type exists, type should not be added into database implicitly by addEntry.
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
if (!typeResult.isHasDictionary()) {
throw new BadRequestException("Entity type does not have a dictionary");
}
var currentVersion = getCurrentVersion(typeResult);
if (removeCurrent) {
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType);
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
} else {
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);
dictionaryService.addEntries(typeId, entries, removeCurrent, ignoreInvalidEntries, dictionaryEntryType);
}
@Override
public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List<String> entries, @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) {
// To check whether the type exists
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
var currentVersion = getCurrentVersion(typeResult);
if (typeResult.isCaseInsensitive()) {
List<String> existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY, null)
.stream()
.map(BaseDictionaryEntry::getValue)
.collect(toList());
entryPersistenceService.deleteEntries(typeId, existing.stream()
.filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase))
.collect(toList()), currentVersion + 1, dictionaryEntryType);
} else {
entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);
dictionaryService.deleteEntries(typeId, entries, dictionaryEntryType);
}
@Override
public void updateTypeValue(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody Type typeRequest) {
validateColor(typeRequest.getHexColor());
validateBoolean(typeRequest.isHint(), "isHint");
validateBoolean(typeRequest.isCaseInsensitive(), "isCaseInsensitive");
String skippedHexColor = typeRequest.getSkippedHexColor();
if (StringUtils.isBlank(skippedHexColor)) { //use the default value
skippedHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getSkippedColor();
typeRequest.setSkippedHexColor(skippedHexColor);
} else {
validateColor(skippedHexColor);
}
String recommendationHexColor = typeRequest.getRecommendationHexColor();
if (StringUtils.isBlank(recommendationHexColor)) { //use the default value
recommendationHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getRecommendationColor();
typeRequest.setRecommendationHexColor(recommendationHexColor);
} else {
validateColor(typeRequest.getRecommendationHexColor());
}
// To check whether the type exists
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
if (typeRequest.getLabel() != null) {
checkForDuplicateLabels(typeResult.getDossierTemplateId(), typeResult.getDossierId(), typeResult.getType(), typeRequest.getLabel());
} else {
typeRequest.setLabel(typeResult.getLabel());
}
dictionaryPersistenceService.updateType(typeId, convert(typeRequest, TypeEntity.class));
if (typeResult.isHint() != typeRequest.isHint() || typeResult.isCaseInsensitive() != typeRequest.isCaseInsensitive() || typeResult
.getRank() != typeRequest.getRank()) {
var currentVersion = getCurrentVersion(typeResult);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
}
dictionaryService.updateTypeValue(typeId, typeRequest);
}
@Override
public Type addType(@RequestBody Type typeRequest) {
if (typeRequest.getDossierTemplateId() == null) {
throw new BadRequestException("Dossier template id does not exist.");
}
if (typeRequest.getLabel() == null || typeRequest.getLabel().isEmpty()) {
String label = humanizedDictionaryType(typeRequest.getType());
typeRequest.setLabel(label);
}
checkForDuplicateLabels(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), typeRequest.getType(), typeRequest
.getLabel());
if (dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), false)
.stream()
.anyMatch(typeResult -> typeRequest.getDossierId() != null && typeResult.getDossierId() != null && typeRequest
.getDossierId()
.equals(typeResult.getDossierId()) && typeRequest.getType()
.equals(typeResult.getType()) && typeRequest.getDossierTemplateId()
.equals(typeResult.getDossierTemplateId()) || typeRequest.getDossierId() == null && typeRequest.getType()
.equals(typeResult.getType()) && typeRequest.getDossierTemplateId()
.equals(typeResult.getDossierTemplateId()))) {
throw new ConflictException("The type already exists, could not be added again.");
}
String color = typeRequest.getHexColor();
validateColor(color);
String skippedHexColor = typeRequest.getSkippedHexColor();
if (StringUtils.isBlank(skippedHexColor)) { //use the default value
skippedHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getSkippedColor();
} else {
validateColor(typeRequest.getSkippedHexColor());
}
String recommendationHexColor = typeRequest.getRecommendationHexColor();
if (StringUtils.isBlank(recommendationHexColor)) { //use the default value
recommendationHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getRecommendationColor();
} else {
validateColor(typeRequest.getRecommendationHexColor());
}
return convert(dictionaryPersistenceService.addType(typeRequest.getType(), typeRequest.getDossierTemplateId(), color, recommendationHexColor, skippedHexColor, typeRequest.getRank(), typeRequest.isHint(), typeRequest.isCaseInsensitive(), typeRequest.isRecommendation(), typeRequest.getDescription(), typeRequest.isAddToDictionaryAction(), typeRequest.getLabel(), typeRequest.getDossierId(), typeRequest.isHasDictionary(), typeRequest.isSystemManaged(), typeRequest.isAutoHideSkipped()), Type.class);
return dictionaryService.addType(typeRequest);
}
@Override
public void deleteType(@PathVariable(TYPE_PARAMETER_NAME) String typeId) {
// NotFoundException would be thrown if the type not found in database.
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
if (typeResult.isSystemManaged()) {
throw new BadRequestException("Can not delete system managed entity type");
}
var currentVersion = getCurrentVersion(typeResult);
dictionaryPersistenceService.deleteType(typeId);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
dictionaryPersistenceService.incrementVersion(typeId);
dictionaryService.deleteType(typeId);
}
@ -266,35 +109,6 @@ public class DictionaryController implements DictionaryResource {
return convert(entries, DictionaryEntry.class);
}
private Set<String> getInvalidEntries(Set<String> entries) {
Predicate<String> isDictionaryEntryNotValid = entry -> DictionaryValidator.validateDictionaryEntry(entry).isPresent();
Predicate<String> isStopword = stopwordService::isStopword;
return entries.stream()
.filter(isDictionaryEntryNotValid.or(isStopword))
.collect(toSet());
}
private void validateBoolean(Boolean bool, String name) {
Optional<String> errorMessage = DictionaryValidator.validateBoolean(bool, name);
if (errorMessage.isPresent()) {
throw new BadRequestException(errorMessage.get());
}
}
private void validateColor(String hexColor) {
Optional<String> errorMessage = DictionaryValidator.validateColor(hexColor);
if (errorMessage.isPresent()) {
throw new BadRequestException(errorMessage.get());
}
}
@Override
public long getVersion(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId) {
@ -325,54 +139,4 @@ public class DictionaryController implements DictionaryResource {
return convert(colorsService.getColors(dossierTemplateId), Colors.class);
}
private String cleanDictionaryEntry(String entry) {
return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
}
private void checkForDuplicateLabels(String dossierTemplateId, String dossierId, String type, String labelToCheck) {
List<TypeEntity> typeResponse = dictionaryPersistenceService.getCumulatedTypes(dossierTemplateId, dossierId, false);
for (TypeEntity res : typeResponse) {
if (res.getDossierId() != null && res.getDossierId().equals(dossierId) && !type.equals(res.getType()) && res
.getDossierTemplateId()
.equals(dossierTemplateId) && labelToCheck.equals(res.getLabel()) || !type.equals(res.getType()) && res
.getDossierTemplateId()
.equals(dossierTemplateId) && labelToCheck.equals(res.getLabel())) {
throw new ConflictException("Label must be unique.");
}
}
}
private long getCurrentVersion(Type typeResult) {
long currentVersion;
if (typeResult.getDossierId() != null) {
currentVersion = getVersionForDossier(typeResult.getDossierId());
} else {
currentVersion = getVersion(typeResult.getDossierTemplateId());
}
return currentVersion;
}
private String humanizedDictionaryType(String label) {
String str = label;
str = str.replaceAll("-+?", " ");
str = str.replaceAll("_+?", " ");
str = str.replaceAll(" +", " ");
StringBuilder strbf = new StringBuilder();
Matcher match = Pattern.compile("([a-z])([a-z]*)", Pattern.CASE_INSENSITIVE).matcher(str);
while (match.find()) {
match.appendReplacement(strbf, match.group(1).toUpperCase() + match.group(2));
}
return match.appendTail(strbf).toString();
}
}

24
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateController.java
View File

@ -1,11 +1,17 @@
package com.iqser.red.service.peristence.v1.server.controller;
import com.iqser.red.service.peristence.v1.server.model.DownloadJob;
import com.iqser.red.service.peristence.v1.server.service.DossierTemplateImportService;
import com.iqser.red.service.peristence.v1.server.service.export.DossierTemplateExportService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CreateOrUpdateDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportDownloadRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.resources.DossierTemplateResource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -23,9 +29,12 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ma
@RequiredArgsConstructor
public class DossierTemplateController implements DossierTemplateResource {
private final DossierTemplateExportService dossierTemplateExportService;
private final DossierTemplateImportService dossierTemplateImportService;
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final DossierTemplateCloneService dossierTemplateCloneService;
@Override
public DossierTemplate createOrUpdateDossierTemplate(@RequestBody CreateOrUpdateDossierTemplateRequest dossierTemplate) {
return convert(dossierTemplatePersistenceService.createOrUpdateDossierTemplate(dossierTemplate), DossierTemplate.class);
@ -51,4 +60,19 @@ public class DossierTemplateController implements DossierTemplateResource {
@RequestBody CloneDossierTemplateRequest cloneDossierTemplateRequest) {
return convert(dossierTemplateCloneService.cloneDossierTemplate(dossierTemplateId,cloneDossierTemplateRequest), DossierTemplate.class);
}
@Override
public JSONPrimitive<String> prepareExportDownload(@RequestBody ExportDownloadRequest request) {
return dossierTemplateExportService.prepareExportDownload(request);
}
public void createExportDownload(@RequestParam String userId, @RequestParam String storageId) {
dossierTemplateExportService.createDownloadArchive(DownloadJob.builder().userId(userId).storageId(storageId).build());
}
public void importDossierTemplate(@RequestBody ImportDossierTemplateRequest request) {
dossierTemplateImportService.importDossierTemplate(request);
}
}

36
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReportTemplateController.java
View File

@ -2,23 +2,21 @@ package com.iqser.red.service.peristence.v1.server.controller;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import com.iqser.red.service.peristence.v1.server.service.ReportTemplateService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUpdateRequest;
import org.apache.commons.io.IOUtils;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateDownload;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUpdateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUploadRequest;
import com.iqser.red.service.persistence.service.v1.api.resources.ReportTemplateResource;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
@ -32,36 +30,12 @@ public class ReportTemplateController implements ReportTemplateResource {
private final StorageService storageService;
private final ReportTemplatePersistenceService reportTemplatePersistenceService;
private final ReportTemplateService reportTemplateService;
public ReportTemplate uploadTemplate(@RequestBody ReportTemplateUploadRequest reportTemplateUploadRequest) {
String templateId = null;
List<ReportTemplateEntity> reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(reportTemplateUploadRequest.getDossierTemplateId());
for (ReportTemplateEntity reportTemplate : reportTemplates) {
if (reportTemplate.getFileName()
.equals(reportTemplateUploadRequest.getFileName()) && reportTemplate.isMultiFileReport() && reportTemplateUploadRequest.isMultiFileReport() || reportTemplate.getFileName()
.equals(reportTemplateUploadRequest.getFileName()) && !reportTemplate.isMultiFileReport() && !reportTemplateUploadRequest.isMultiFileReport()) {
templateId = reportTemplate.getTemplateId();
}
}
String storageId = StorageIdUtils.getReportStorageId(reportTemplateUploadRequest.getDossierTemplateId(), reportTemplateUploadRequest.getFileName());
storageService.storeObject(storageId, new ByteArrayInputStream(reportTemplateUploadRequest.getTemplate()));
if (templateId != null) {
reportTemplatePersistenceService.updateTemplate(reportTemplateUploadRequest.getDossierTemplateId(), templateId, ReportTemplateUpdateRequest.builder()
.fileName(reportTemplateUploadRequest.getFileName())
.multiFileReport(reportTemplateUploadRequest.isMultiFileReport())
.activeByDefault(reportTemplateUploadRequest.isActiveByDefault())
.build());
} else {
templateId = UUID.randomUUID().toString();
reportTemplatePersistenceService.insert(reportTemplateUploadRequest.getDossierTemplateId(), templateId, storageId, reportTemplateUploadRequest.getFileName(),
reportTemplateUploadRequest.isActiveByDefault(), reportTemplateUploadRequest.isMultiFileReport());
}
return convert(reportTemplatePersistenceService.find(templateId), ReportTemplate.class);
return reportTemplateService.uploadTemplate(reportTemplateUploadRequest);
}

284
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java Normal file
View File

@ -0,0 +1,284 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities;
import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
@Slf4j
@Service
@RequiredArgsConstructor
public class DictionaryService {
private final EntryPersistenceService entryPersistenceService;
private final DictionaryPersistenceService dictionaryPersistenceService;
private final ColorsService colorsService;
private final StopwordService stopwordService;
public Type addType(Type typeRequest) {
if (typeRequest.getDossierTemplateId() == null) {
throw new BadRequestException("Dossier template id does not exist.");
}
if (typeRequest.getLabel() == null || typeRequest.getLabel().isEmpty()) {
String label = humanizedDictionaryType(typeRequest.getType());
typeRequest.setLabel(label);
}
checkForDuplicateLabels(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), typeRequest.getType(), typeRequest
.getLabel());
if (this.checkForExistingType(typeRequest)) {
throw new ConflictException("The type already exists, could not be added again.");
}
String color = typeRequest.getHexColor();
validateColor(color);
String skippedHexColor = typeRequest.getSkippedHexColor();
if (StringUtils.isBlank(skippedHexColor)) { //use the default value
skippedHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getSkippedColor();
} else {
validateColor(typeRequest.getSkippedHexColor());
}
String recommendationHexColor = typeRequest.getRecommendationHexColor();
if (StringUtils.isBlank(recommendationHexColor)) { //use the default value
recommendationHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getRecommendationColor();
} else {
validateColor(typeRequest.getRecommendationHexColor());
}
return convert(dictionaryPersistenceService.addType(typeRequest.getType(), typeRequest.getDossierTemplateId(), color, recommendationHexColor, skippedHexColor, typeRequest.getRank(), typeRequest.isHint(), typeRequest.isCaseInsensitive(), typeRequest.isRecommendation(), typeRequest.getDescription(), typeRequest.isAddToDictionaryAction(), typeRequest.getLabel(), typeRequest.getDossierId(), typeRequest.isHasDictionary(), typeRequest.isSystemManaged(), typeRequest.isAutoHideSkipped()), Type.class);
}
public void updateTypeValue(String typeId, Type typeRequest) {
validateColor(typeRequest.getHexColor());
validateBoolean(typeRequest.isHint(), "isHint");
validateBoolean(typeRequest.isCaseInsensitive(), "isCaseInsensitive");
String skippedHexColor = typeRequest.getSkippedHexColor();
if (StringUtils.isBlank(skippedHexColor)) { //use the default value
skippedHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getSkippedColor();
typeRequest.setSkippedHexColor(skippedHexColor);
} else {
validateColor(skippedHexColor);
}
String recommendationHexColor = typeRequest.getRecommendationHexColor();
if (StringUtils.isBlank(recommendationHexColor)) { //use the default value
recommendationHexColor = colorsService.getColors(typeRequest.getDossierTemplateId()).getRecommendationColor();
typeRequest.setRecommendationHexColor(recommendationHexColor);
} else {
validateColor(typeRequest.getRecommendationHexColor());
}
// To check whether the type exists
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
if (typeRequest.getLabel() != null) {
checkForDuplicateLabels(typeResult.getDossierTemplateId(), typeResult.getDossierId(), typeResult.getType(), typeRequest.getLabel());
} else {
typeRequest.setLabel(typeResult.getLabel());
}
dictionaryPersistenceService.updateType(typeId, convert(typeRequest, TypeEntity.class));
if (typeResult.isHint() != typeRequest.isHint() || typeResult.isCaseInsensitive() != typeRequest.isCaseInsensitive() || typeResult
.getRank() != typeRequest.getRank()) {
var currentVersion = getCurrentVersion(typeResult);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
}
}
public void addEntries(String typeId, List<String> entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType) {
Set<String> cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet());
if (CollectionUtils.isEmpty(entries)) {
throw new BadRequestException("Entry list is empty.");
}
var invalidEntries = getInvalidEntries(cleanEntries);
if (!ignoreInvalidEntries && CollectionUtils.isNotEmpty(invalidEntries)) {
throw new BadRequestException("Error(s) validating dictionary entries:\\n" + String.join("\\n", invalidEntries));
} else {
cleanEntries.removeAll(invalidEntries);
}
// To check whether the type exists, type should not be added into database implicitly by addEntry.
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
if (!typeResult.isHasDictionary()) {
throw new BadRequestException("Entity type does not have a dictionary");
}
var currentVersion = getCurrentVersion(typeResult);
if (removeCurrent) {
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType);
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
} else {
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);
}
public void deleteEntries(String typeId, List<String> entries, DictionaryEntryType dictionaryEntryType) {
// To check whether the type exists
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
var currentVersion = getCurrentVersion(typeResult);
if (typeResult.isCaseInsensitive()) {
List<String> existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY, null)
.stream()
.map(BaseDictionaryEntry::getValue)
.collect(toList());
entryPersistenceService.deleteEntries(typeId, existing.stream()
.filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase))
.collect(toList()), currentVersion + 1, dictionaryEntryType);
} else {
entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);
}
public void deleteType( String typeId) {
// NotFoundException would be thrown if the type not found in database.
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
if (typeResult.isSystemManaged()) {
throw new BadRequestException("Can not delete system managed entity type");
}
var currentVersion = getCurrentVersion(typeResult);
dictionaryPersistenceService.deleteType(typeId);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
dictionaryPersistenceService.incrementVersion(typeId);
}
private void validateBoolean(Boolean bool, String name) {
Optional<String> errorMessage = DictionaryValidator.validateBoolean(bool, name);
if (errorMessage.isPresent()) {
throw new BadRequestException(errorMessage.get());
}
}
private void validateColor(String hexColor) {
Optional<String> errorMessage = DictionaryValidator.validateColor(hexColor);
if (errorMessage.isPresent()) {
throw new BadRequestException(errorMessage.get());
}
}
private void checkForDuplicateLabels(String dossierTemplateId, String dossierId, String type, String labelToCheck) {
List<TypeEntity> typeResponse = dictionaryPersistenceService.getCumulatedTypes(dossierTemplateId, dossierId, false);
for (TypeEntity res : typeResponse) {
if (res.getDossierId() != null && res.getDossierId().equals(dossierId) && !type.equals(res.getType()) && res
.getDossierTemplateId()
.equals(dossierTemplateId) && labelToCheck.equals(res.getLabel()) || !type.equals(res.getType()) && res
.getDossierTemplateId()
.equals(dossierTemplateId) && labelToCheck.equals(res.getLabel())) {
throw new ConflictException("Label must be unique.");
}
}
}
private String humanizedDictionaryType(String label) {
String str = label;
str = str.replaceAll("-+?", " ");
str = str.replaceAll("_+?", " ");
str = str.replaceAll(" +", " ");
StringBuilder strbf = new StringBuilder();
Matcher match = Pattern.compile("([a-z])([a-z]*)", Pattern.CASE_INSENSITIVE).matcher(str);
while (match.find()) {
match.appendReplacement(strbf, match.group(1).toUpperCase() + match.group(2));
}
return match.appendTail(strbf).toString();
}
private Set<String> getInvalidEntries(Set<String> entries) {
Predicate<String> isDictionaryEntryNotValid = entry -> DictionaryValidator.validateDictionaryEntry(entry).isPresent();
Predicate<String> isStopword = stopwordService::isStopword;
return entries.stream()
.filter(isDictionaryEntryNotValid.or(isStopword))
.collect(toSet());
}
private String cleanDictionaryEntry(String entry) {
return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
}
private long getCurrentVersion(Type typeResult) {
long currentVersion;
if (typeResult.getDossierId() != null) {
currentVersion = dictionaryPersistenceService.getVersionForDossier(typeResult.getDossierId());
} else {
currentVersion = dictionaryPersistenceService.getVersion(typeResult.getDossierTemplateId());
}
return currentVersion;
}
public boolean checkForExistingType(Type typeRequest) {
return dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), false)
.stream()
.anyMatch(typeResult -> typeRequest.getDossierId() != null && typeResult.getDossierId() != null
&& typeRequest.getDossierId().equals(typeResult.getDossierId())
&& typeRequest.getType().equals(typeResult.getType())
&& typeRequest.getDossierTemplateId().equals(typeResult.getDossierTemplateId())
|| typeRequest.getDossierId() == null && typeRequest.getType().equals(typeResult.getType())
&& typeRequest.getDossierTemplateId().equals(typeResult.getDossierTemplateId()));
}
}

410
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateImportService.java Normal file
View File

@ -0,0 +1,410 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Watermark;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatusInfo;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportFilename;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportTemplateResult;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@Slf4j
@Service
@RequiredArgsConstructor
public class DossierTemplateImportService {
private static final int THRESHOLD_ENTRIES = 10000;
private static final int THRESHOLD_SIZE = 1000000000; // 1 GB
private static final double THRESHOLD_RATIO = 10;
private final DossierTemplateRepository dossierTemplateRepository;
private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
private final RulesPersistenceService rulesPersistenceService;
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
private final ColorsService colorsService;
private final DossierStatusPersistenceService dossierStatusPersistenceService;
private final WatermarkService watermarkService;
private final ReportTemplateService reportTemplateService;
private final DictionaryService dictionaryService;
private final ObjectMapper objectMapper = new ObjectMapper();
public void importDossierTemplate(@RequestBody ImportDossierTemplateRequest request) {
ImportTemplateResult result = this.handleArchive(request);
if (result == null) {
throw new BadRequestException("Error while reading the archive");
}
importDossierTemplate(result);
}
public void importDossierTemplate(ImportTemplateResult request) {
String dossierTemplateId;
var dossierTemplateMeta = request.getDossierTemplate();
var dossierTemplate = dossierTemplateRepository.findByIdAndNotDeleted(request.getDossierTemplateId());
if (dossierTemplate.isPresent() && request.isUpdateExistingTemplate()) {
dossierTemplateId = dossierTemplate.get().getId();
// override the existing dossier template
updateDossierTemplateMeta(dossierTemplate.get(), dossierTemplateMeta, request.getUserId());
// set rules
rulesPersistenceService.setRules(request.getRuleSet(), dossierTemplateId);
dossierTemplate.get().setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(dossierTemplate.get()).name()));
} else {
// creates new dossier template
if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
throw new ConflictException("DossierTemplate name must be set");
}
dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName());
DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
dossierTemplateEntity.setId(UUID.randomUUID().toString());
// order is important
BeanUtils.copyProperties(dossierTemplateMeta, dossierTemplateEntity);
dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
dossierTemplateEntity.setCreatedBy(request.getUserId());
//set rules
rulesPersistenceService.setRules(request.getRuleSet(), dossierTemplateEntity.getId());
var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity);
loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate));
dossierTemplateId = loadedDossierTemplate.getId();
}
// set legal basis
if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
}
//set dossier attributes
if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
dossierAttributeConfigPersistenceService.setDossierAttributesConfig(dossierTemplateId,
convert(request.getDossierAttributesConfigs(), DossierAttributeConfigEntity.class));
}
// dossier status
if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
updateDossierStates(dossierTemplateId, request.getDossierStatusInfos());
}
//set file attributes
if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
fileAttributeConfigPersistenceService.setFileAttributesConfig(dossierTemplateId,
convert(request.getFileAttributesConfigs(), FileAttributeConfigEntity.class));
}
if(request.getFileAttributesGeneralConfiguration() != null) {
fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
convert(request.getFileAttributesGeneralConfiguration(), FileAttributesGeneralConfigurationEntity.class));
}
//set report templates
if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
request.getReportTemplateUploadRequests().forEach(reportRequest -> {
reportRequest.setDossierTemplateId(dossierTemplateId);
reportTemplateService.uploadTemplate(reportRequest);
});
}
// set colors
if (request.getColors() != null) {
ColorsEntity colorsEntity = convert(request.getColors(), ColorsEntity.class);
colorsEntity.setDossierTemplateId(dossierTemplateId);
colorsService.saveColors(colorsEntity);
}
// set watermarks
if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
request.getWatermarks().forEach(watermark -> {
try {
watermarkService.createOrUpdateWatermark(watermark);
} catch (BadRequestException e) {
log.debug(" Conflict while validating watermark: " + watermark.getId());
}
});
}
// set the types
// should delete existing types? in case of conflicts
for (var type: request.getTypes()) {
type.setDossierTemplateId(dossierTemplateId);
String typeId = type.getTypeId();
if (dictionaryService.checkForExistingType(type)) {
dictionaryService.updateTypeValue(type.getTypeId(), type);
} else {
var returnedType = dictionaryService.addType(type);
typeId = returnedType.getTypeId();
}
if (request.getEntries() != null && !request.getEntries().isEmpty()
&& !request.getEntries().get(type.getType()).isEmpty()) {
dictionaryService.addEntries(typeId, request.getEntries().get(type.getType()), true, true, DictionaryEntryType.ENTRY);
}
if (request.getFalsePositives() != null && !request.getFalsePositives().isEmpty()
&& !request.getFalsePositives().get(type.getType()).isEmpty()) {
dictionaryService.addEntries(typeId, request.getFalsePositives().get(type.getType()), true, true, DictionaryEntryType.FALSE_POSITIVE);
}
if (request.getFalseRecommendations() != null && !request.getFalseRecommendations().isEmpty()
&& !request.getFalseRecommendations().get(type.getType()).isEmpty()) {
dictionaryService.addEntries(typeId, request.getFalseRecommendations().get(type.getType()), true, true, DictionaryEntryType.FALSE_RECOMMENDATION);
}
}
}
private void updateDossierTemplateMeta(DossierTemplateEntity dossierTemplateEntity, DossierTemplate dossierTemplate, String userId) {
if (!dossierTemplateEntity.getName().equalsIgnoreCase(dossierTemplate.getName())) {
dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplate.getName());
}
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
dossierTemplateEntity.setName(dossierTemplate.getName());
dossierTemplateEntity.setDescription(dossierTemplate.getDescription());
dossierTemplateEntity.setDateModified(now);
dossierTemplateEntity.setModifiedBy(userId);
dossierTemplateEntity.setValidFrom(dossierTemplate.getValidFrom() == null ? dossierTemplateEntity.getValidFrom() : dossierTemplate.getValidFrom());
dossierTemplateEntity.setValidTo(dossierTemplate.getValidTo() == null ? dossierTemplateEntity.getValidTo() : dossierTemplate.getValidTo());
dossierTemplateEntity.setDownloadFileTypes(dossierTemplate.getDownloadFileTypes() == null ||
dossierTemplate.getDownloadFileTypes().isEmpty() ? dossierTemplateEntity.getDownloadFileTypes() : dossierTemplate.getDownloadFileTypes());
}
private void updateDossierStates(String dossierTemplateId, List<DossierStatusInfo> dossierStatusInfoList) {
dossierStatusInfoList.forEach(state -> {
var dossierStatusRequest = CreateOrUpdateDossierStatusRequest.builder()
.dossierStatusId(state.getId())
.name(state.getName())
.description(state.getDescription())
.color(state.getColor())
.dossierTemplateId(dossierTemplateId)
.rank(state.getRank())
.build();
try {
dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
} catch (BadRequestException e) {
log.debug(" Empty name for dossier status id " + state.getId());
}catch (ConflictException e) {
log.debug(" Conflict detected for dossier status id: " + state.getId());
} catch (NotFoundException e) {
log.debug(" Not found dossier status id " + state.getId());
}
});
}
public ImportTemplateResult handleArchive(ImportDossierTemplateRequest request) {
objectMapper.registerModule(new JavaTimeModule());
File tempFile;
try {
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
tempFile = Files.createTempFile(UUID.randomUUID().toString(), ".zip", attr).toFile();
IOUtils.write(request.getArchive(), new FileOutputStream(tempFile));
var importTemplateResult = new ImportTemplateResult();
importTemplateResult.setDossierTemplateId(request.getDossierTemplateId());
importTemplateResult.setUserId(request.getUserId());
importTemplateResult.setUpdateExistingTemplate(request.isUpdateExistingDossierTemplate());
try (FileInputStream fis = new FileInputStream(tempFile); BufferedInputStream bis = new BufferedInputStream(fis);
ZipArchiveInputStream zis = new ZipArchiveInputStream(bis)) {
int totalSizeArchive = 0;
int totalEntryArchive = 0;
ZipArchiveEntry ze;
Map<String, ReportTemplate> reportTemplateMap = new HashMap<>();
Map<String, ByteArrayOutputStream> reportTemplateBytesMap = new HashMap<>();
List<String> reportTemplateFilenameList = new ArrayList<>();
Map<String, List<String>> typeEntriesMap = new HashMap<>();
Map<String, List<String>> typeFalsePositivesMap = new HashMap<>();
Map<String, List<String>> typeFalseRecommendationsMap = new HashMap<>();
while ((ze = zis.getNextZipEntry()) != null) {
log.debug("---> " + ze.getName() + " ---- " + ze.isDirectory());
totalEntryArchive++;
if (ze.isUnixSymlink()) {
continue;
}
if (!ze.isDirectory()) {
var bos = new ByteArrayOutputStream();
var buffer = new byte[2048];
var nBytes = 0;
int totalSizeEntry = 0;
while ((nBytes = zis.read(buffer)) > 0) {
bos.write(buffer, 0, nBytes);
totalSizeEntry += nBytes;
totalSizeArchive += nBytes;
double compressionRatio = (float) totalSizeEntry / ze.getCompressedSize();
if (compressionRatio > THRESHOLD_RATIO) {
// ratio between compressed and uncompressed data is highly suspicious, looks like a Zip Bomb Attack
throw new BadRequestException("ZIP-Bomb detected.");
}
}
if (totalSizeArchive > THRESHOLD_SIZE) {
// the uncompressed data size is too much for the application resource capacity
throw new BadRequestException("ZIP-Bomb detected.");
}
if (totalEntryArchive > THRESHOLD_ENTRIES) {
// too much entries in this archive, can lead to inodes exhaustion of the system
throw new BadRequestException("ZIP-Bomb detected.");
}
var bytes = bos.toByteArray();
if (ze.getName().contains(ExportFilename.DOSSIER_TEMPLATE_META.getFilename())) {
DossierTemplate dossierTemplate = objectMapper.readValue(bytes, DossierTemplate.class);
importTemplateResult.setDossierTemplate(dossierTemplate);
} else if (ze.getName().contains(ExportFilename.WATERMARK.getFilename())) {
List<Watermark> watermarkList = objectMapper.readValue(bytes, new TypeReference<List<Watermark>>() {
});
importTemplateResult.getWatermarks().addAll(watermarkList);
} else if (ze.getName().contains(ExportFilename.COLORS.getFilename())) {
Colors colors = objectMapper.readValue(bytes, Colors.class);
importTemplateResult.setColors(colors);
} else if (ze.getName().contains(ExportFilename.DOSSIER_STATUS.getFilename())) {
List<DossierStatusInfo> dossierStatusInfoList = objectMapper.readValue(bytes, new TypeReference<List<DossierStatusInfo>>() {
});
importTemplateResult.getDossierStatusInfos().addAll(dossierStatusInfoList);
} else if (ze.getName().contains(ExportFilename.DOSSIER_ATTRIBUTES_CONFIG.getFilename())) {
List<DossierAttributeConfig> dossierAttributeConfigs = objectMapper.readValue(bytes, new TypeReference<List<DossierAttributeConfig>>() {
});
importTemplateResult.getDossierAttributesConfigs().addAll(dossierAttributeConfigs);
} else if (ze.getName().contains(ExportFilename.FILE_ATTRIBUTE_CONFIG.getFilename())) {
List<FileAttributeConfig> fileAttributeConfigs = objectMapper.readValue(bytes, new TypeReference<List<FileAttributeConfig>>() {
});
importTemplateResult.getFileAttributesConfigs().addAll(fileAttributeConfigs);
} else if (ze.getName().contains(ExportFilename.LEGAL_BASIS.getFilename())) {
List<LegalBasis> legalBasisList = objectMapper.readValue(bytes, new TypeReference<List<LegalBasis>>() {
});
importTemplateResult.getLegalBases().addAll(legalBasisList);
} else if (ze.getName().contains(ExportFilename.FILE_ATTRIBUTE_GENERAL_CONFIG.getFilename())) {
FileAttributesGeneralConfiguration fileAttributesGeneralConfiguration = objectMapper.readValue(bytes, FileAttributesGeneralConfiguration.class);
importTemplateResult.setFileAttributesGeneralConfiguration(fileAttributesGeneralConfiguration);
} else if (ze.getName().contains(ExportFilename.RULES.getFilename())) {
String rules = objectMapper.readValue(bytes, String.class);
importTemplateResult.setRuleSet(rules);
} else if (ze.getName().contains(ExportFilename.DOSSIER_TYPE.getFilename())) {
Type type = objectMapper.readValue(bytes, Type.class);
importTemplateResult.getTypes().add(type);
} else if (ze.getName().contains(ExportFilename.ENTRIES.getFilename())) {
List<String> entries = this.readEntries(bytes);
typeEntriesMap.put(this.getTypeName(ze.getName()), entries);
importTemplateResult.setEntries(typeEntriesMap);
} else if (ze.getName().contains(ExportFilename.FALSE_POSITIVES.getFilename())) {
List<String> falsePositives = this.readEntries(bytes);
typeFalsePositivesMap.put(this.getTypeName(ze.getName()), falsePositives);
importTemplateResult.setFalsePositives(typeFalsePositivesMap);
} else if (ze.getName().contains(ExportFilename.FALSE_RECOMMENDATION.getFilename())) {
List<String> falseRecommendations = this.readEntries(bytes);
typeFalseRecommendationsMap.put(this.getTypeName(ze.getName()), falseRecommendations);
importTemplateResult.setFalseRecommendations(typeFalseRecommendationsMap);
} else if (ze.getName().contains("reportTemplateList.json")) {
var reportTemplateList = objectMapper.readValue(bytes, new TypeReference<List<ReportTemplate>>() {
});
reportTemplateMap = reportTemplateList.stream()
.collect(Collectors.toMap(ReportTemplate::getFileName, Function.identity()));
reportTemplateFilenameList = reportTemplateList.stream().map(rt -> rt.getFileName()).collect(Collectors.toList());
} else {
reportTemplateBytesMap.put(ze.getName(), bos);
}
bos.close();
}
}
List<ReportTemplateUploadRequest> reportTemplateUploadRequests = new ArrayList<>();
for (var reportZe : reportTemplateBytesMap.entrySet()) {
if (reportTemplateFilenameList.contains(reportZe.getKey())) {
var report = reportTemplateMap.get(reportZe.getKey());
importTemplateResult.getReportTemplateUploadRequests().add(ReportTemplateUploadRequest.builder()
.fileName(report.getFileName())
.activeByDefault(report.isActiveByDefault())
.multiFileReport(report.isMultiFileReport())
.dossierTemplateId(request.getDossierTemplateId())
.template(reportZe.getValue().toByteArray())
.build());
}
importTemplateResult.getReportTemplateUploadRequests().addAll(reportTemplateUploadRequests);
}
return importTemplateResult;
} finally {
if (tempFile != null) {
boolean isDeleted = tempFile.delete();
if (!isDeleted) {
log.debug("tempFile could not be deleted");
}
}
}
} catch (IOException e) {
log.debug("exception: ", e);
} catch (BadRequestException e) {
log.debug("exception: ", e);
}
return null;
}
private List<String> readEntries(byte[] bytes ) {
ByteArrayInputStream bInput = new ByteArrayInputStream(bytes);
DataInputStream in = new DataInputStream(bInput);
List<String> entries = new ArrayList<>();
try {
while (in.available() > 0) {
String entry = in.readUTF();
entries.add(entry);
}
} catch (IOException e) {
log.debug("exception: ", e);
}
return entries;
}
private String getTypeName(String filename) {
var index = filename.indexOf('/');
return filename.substring(0, index);
}
}

61
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReportTemplateService.java Normal file
View File

@ -0,0 +1,61 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUpdateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUploadRequest;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.util.List;
import java.util.UUID;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@Slf4j
@Service
@RequiredArgsConstructor
public class ReportTemplateService {
private final StorageService storageService;
private final ReportTemplatePersistenceService reportTemplatePersistenceService;
public ReportTemplate uploadTemplate(ReportTemplateUploadRequest reportTemplateUploadRequest) {
String templateId = null;
List<ReportTemplateEntity> reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(reportTemplateUploadRequest
.getDossierTemplateId());
for (ReportTemplateEntity reportTemplate : reportTemplates) {
if (reportTemplate.getFileName()
.equals(reportTemplateUploadRequest.getFileName()) && reportTemplate.isMultiFileReport() && reportTemplateUploadRequest
.isMultiFileReport() || reportTemplate.getFileName()
.equals(reportTemplateUploadRequest.getFileName()) && !reportTemplate.isMultiFileReport() && !reportTemplateUploadRequest
.isMultiFileReport()) {
templateId = reportTemplate.getTemplateId();
}
}
String storageId = StorageIdUtils.getReportStorageId(reportTemplateUploadRequest.getDossierTemplateId(), reportTemplateUploadRequest
.getFileName());
storageService.storeObject(storageId, new ByteArrayInputStream(reportTemplateUploadRequest.getTemplate()));
if (templateId != null) {
reportTemplatePersistenceService.updateTemplate(reportTemplateUploadRequest.getDossierTemplateId(), templateId, ReportTemplateUpdateRequest
.builder()
.fileName(reportTemplateUploadRequest.getFileName())
.multiFileReport(reportTemplateUploadRequest.isMultiFileReport())
.activeByDefault(reportTemplateUploadRequest.isActiveByDefault())
.build());
} else {
templateId = UUID.randomUUID().toString();
reportTemplatePersistenceService.insert(reportTemplateUploadRequest.getDossierTemplateId(), templateId, storageId, reportTemplateUploadRequest
.getFileName(), reportTemplateUploadRequest.isActiveByDefault(), reportTemplateUploadRequest.isMultiFileReport());
}
return convert(reportTemplatePersistenceService.find(templateId), ReportTemplate.class);
}
}

207
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/DossierTemplateExportService.java Normal file
View File

@ -0,0 +1,207 @@
package com.iqser.red.service.peristence.v1.server.service.export;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.model.DownloadJob;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.utils.FileSystemBackedArchiver;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Watermark;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatusInfo;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportDownloadRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportFilename;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import java.io.*;
import java.util.List;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@Slf4j
@Service
@RequiredArgsConstructor
public class DossierTemplateExportService {
private static final String JSON_EXT = ".json";
private static final String TXT_EXT = ".txt";
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final DownloadStatusPersistenceService downloadStatusPersistenceService;
private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
private final RulesPersistenceService rulesPersistenceService;
private final FileManagementStorageService fileManagementStorageService;
private final ColorsService colorsService;
private final EntryPersistenceService entryPersistenceService;
private final ObjectMapper objectMapper = new ObjectMapper();
private final RabbitTemplate rabbitTemplate;
public JSONPrimitive<String> prepareExportDownload(ExportDownloadRequest request) {
var mimeType = "application/zip";
String downloadFilename = request.getDossierTemplateId() + ".zip";
String storageId = StorageIdUtils.getStorageId(request.getUserId(), request.getDossierTemplateId());
downloadStatusPersistenceService.createStatus(request.getUserId(), storageId, downloadFilename, mimeType);
addToExportDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).build(), 1);
return new JSONPrimitive<>(storageId);
}
@SneakyThrows
public void createDownloadArchive(DownloadJob downloadJob) {
objectMapper.registerModule(new JavaTimeModule());
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId());
String dossierTemplateId = extractDossierTemplateId(downloadStatus.getFilename());
var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate((dossierTemplateId));
try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {
// add dossier template name and meta data
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.DOSSIER_TEMPLATE_META, JSON_EXT),
objectMapper.writeValueAsBytes(convert(dossierTemplate, DossierTemplate.class))));
//add watermark json file
var watermarkList = dossierTemplate.getWatermarkConfigs();
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.WATERMARK, JSON_EXT),
objectMapper.writeValueAsBytes(convert(watermarkList, Watermark.class))));
//add colors json file
var colors = colorsService.getColors(dossierTemplateId);
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.COLORS, JSON_EXT),
objectMapper.writeValueAsBytes(convert(colors, Colors.class))));
// add dossier statuses
var dossierStatusList = dossierTemplate.getDossierStatusList();
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.DOSSIER_STATUS, JSON_EXT),
objectMapper.writeValueAsBytes(convert(dossierStatusList, DossierStatusInfo.class))));
// add dossier attributes config json
var dossierAttributesConfig = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.DOSSIER_ATTRIBUTES_CONFIG, JSON_EXT),
objectMapper.writeValueAsBytes(convert(dossierAttributesConfig, DossierAttributeConfig.class))));
// add file attribute configs
var fileAttributeConfigList = dossierTemplate.getFileAttributeConfigs();
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.FILE_ATTRIBUTE_CONFIG, JSON_EXT),
objectMapper.writeValueAsBytes(convert(fileAttributeConfigList, FileAttributeConfig.class))));
// add legal basis mapping
var legalBasisMappingList = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId);
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.LEGAL_BASIS, JSON_EXT),
objectMapper.writeValueAsBytes(convert(legalBasisMappingList, LegalBasis.class))));
// add file attributes general configuration
var fileAttributesGeneralConfig = fileAttributeConfigPersistenceService.getFileAttributesGeneralConfiguration(dossierTemplateId);
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.FILE_ATTRIBUTE_GENERAL_CONFIG, JSON_EXT),
objectMapper.writeValueAsBytes(convert(fileAttributesGeneralConfig, FileAttributesGeneralConfiguration.class))));
// add rule set
var ruleSet = rulesPersistenceService.getRules(dossierTemplateId);
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.RULES, TXT_EXT),
objectMapper.writeValueAsBytes(ruleSet.getValue())));
//N files with the related report templates
var reportTemplateList = dossierTemplate.getReportTemplates();
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, getFilename(ExportFilename.REPORT_TEMPLATE, JSON_EXT),
objectMapper.writeValueAsBytes(convert(reportTemplateList, ReportTemplate.class))));
reportTemplateList.forEach(reportTemplate -> {
var report = fileManagementStorageService.getStoredObjectBytes(reportTemplate.getStorageId());
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(null, reportTemplate.getFileName(), report));
});
// for every type 1 folder with:
// 1 json file containing meta info of the type
// and 1 txt file for every type: entries, false positives and false recommendation
var dossierTypes = dossierTemplate.getDossierTypes();
for (TypeEntity typeEntity : dossierTypes) {
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(typeEntity.getType(), getFilename(ExportFilename.DOSSIER_TYPE, JSON_EXT),
objectMapper.writeValueAsBytes(convert(typeEntity, Type.class))));
var entriesValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.ENTRY, null)
.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList());
var falsePositiveValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.FALSE_POSITIVE, null)
.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList());
var falseRecommendationValuesList = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.FALSE_RECOMMENDATION, null)
.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList());
writeEntriesListToFile(fileSystemBackedArchiver, entriesValuesList, typeEntity.getType(), getFilename(ExportFilename.ENTRIES, TXT_EXT));
writeEntriesListToFile(fileSystemBackedArchiver, falsePositiveValuesList, typeEntity.getType(),getFilename(ExportFilename.FALSE_POSITIVES, TXT_EXT));
writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(),getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT));
}
storeZipFile(downloadStatus.getStorageId(), fileSystemBackedArchiver);
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
} catch (JsonProcessingException e) {
log.debug("fail ", e);
throw new BadRequestException("Error creating the archive"); // update error handling
}
}
private String getFilename(ExportFilename exportFilename, String extension) {
return exportFilename.getFilename() + extension;
}
private void writeEntriesListToFile(FileSystemBackedArchiver fileSystemBackedArchiver, List<String> entriesValuesList, String folderName, String filename) {
try {
ByteArrayOutputStream bt = new ByteArrayOutputStream();
DataOutputStream dt = new DataOutputStream(bt);
for(String entry : entriesValuesList) {
dt.writeUTF(entry);
}
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folderName, filename, bt.toByteArray()));
dt.close();
bt.close();
} catch (IOException e) {
log.debug("Error writing values to files");
}
}
private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) {
long start = System.currentTimeMillis();
fileManagementStorageService.storeObject(storageId, fileSystemBackedArchiver.toInputStream());
log.info("Successfully stored zip for downloadId {}, took {}", storageId, System.currentTimeMillis() - start);
}
private String extractDossierTemplateId(String fileName) {
var lastIndexOfDot = fileName.lastIndexOf(".");
return fileName.substring(0, lastIndexOfDot);
}
private void addToExportDownloadQueue(DownloadJob downloadJob, int priority) {
try {
rabbitTemplate.convertAndSend(MessagingConfiguration.DOWNLOAD_QUEUE, objectMapper.writeValueAsString(downloadJob), message -> {
message.getMessageProperties().setPriority(priority);
return message;
});
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
}

30
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/export/ExportDownloadMessageReceiver.java Normal file
View File

@ -0,0 +1,30 @@
package com.iqser.red.service.peristence.v1.server.service.export;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.model.DownloadJob;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
@RabbitListener(queues = MessagingConfiguration.EXPORT_DOWNLOAD_QUEUE)
public class ExportDownloadMessageReceiver {
private final DossierTemplateExportService dossierTemplateService;
private final ObjectMapper objectMapper;
@RabbitHandler
public void receive(String in) throws JsonProcessingException {
DownloadJob downloadJob = objectMapper.readValue(in, DownloadJob.class);
log.info("Preparing export download for userId: {} and storageId: {}", downloadJob.getUserId(), downloadJob.getStorageId());
dossierTemplateService.createDownloadArchive(downloadJob);
}
}

5
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/StorageIdUtils.java
View File

@ -17,6 +17,11 @@ public final class StorageIdUtils {
return userId + "/" + dossierId + "/" + filename;
}
public static String getStorageId(String userId, String dossierTemplateId) {
return userId + "/" + dossierTemplateId;
}
public static String getReportStorageId(String dossierTemplateId, String fileName) {
return dossierTemplateId + "/" + fileName;
}

222
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java
View File

@ -1,9 +1,12 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.integration.client.*;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.peristence.v1.server.service.export.ExportDownloadMessageReceiver;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.*;
@ -14,18 +17,22 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ExportDownloadRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
import com.iqser.red.storage.commons.service.StorageService;
import feign.FeignException;
import lombok.SneakyThrows;
import org.junit.Test;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.List;
import java.util.*;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static org.assertj.core.api.Assertions.assertThat;
@ -62,6 +69,30 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
@Autowired
private DossierStatusClient dossierStatusClient;
@Autowired
private DownloadClient downloadClient;
@Autowired
private ObjectMapper objectMapper;
@Autowired
private ExportDownloadMessageReceiver exportDownloadReportMessageReceiver;
@Autowired
private StorageService storageService;
@Test
public void testDownload() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
// test the export of dossier template
dossierTemplateClient.prepareExportDownload(ExportDownloadRequest.builder()
.userId("1")
.dossierTemplateId(dossierTemplate.getId())
.build());
var statuses = downloadClient.getDownloadStatus("1");
assertThat(statuses).isNotEmpty();
}
@Test
public void testDossierTemplate() {
@ -115,6 +146,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
@Test
@SneakyThrows
public void testCloneDossierTemplate() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
@ -285,4 +317,190 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
.isEqualTo(watermarkClient.getWatermarksForDossierTemplateId(clonedDT.getId()).get(0).getName());
}
@SneakyThrows
@Test
public void testExportDossierTemplate() {
// TenantContext.setTenantId(DEFAULT_TENANT);
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
setupDossierTemplate(dossierTemplate);
dossierTemplateClient.prepareExportDownload(ExportDownloadRequest.builder()
.userId("1")
.dossierTemplateId(dossierTemplate.getId())
.build());
var statuses = downloadClient.getDownloadStatus("1");
assertThat(statuses).isNotEmpty();
// var downloadJob = DownloadJob.builder()
// .storageId(StorageIdUtils.getStorageId("1", dossierTemplate.getId()))
// .userId("1").build();
// exportDownloadReportMessageReceiver.receive(objectMapper.writeValueAsString(downloadJob));
String storageId = StorageIdUtils.getStorageId("1", dossierTemplate.getId());
dossierTemplateClient.createExportDownload("1", storageId);
statuses = downloadClient.getDownloadStatus("1");
assertThat(statuses).isNotEmpty();
assertThat(statuses.get(0).getStatus()).isEqualTo(DownloadStatusValue.READY);
}
@Test
@SneakyThrows
public void testImportDossierTemplateUpdateExisting() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
setupDossierTemplate(dossierTemplate);
// prepare an archive
dossierTemplateClient.prepareExportDownload(ExportDownloadRequest.builder()
.userId("1")
.dossierTemplateId(dossierTemplate.getId())
.build());
String storageId = StorageIdUtils.getStorageId("1", dossierTemplate.getId());
dossierTemplateClient.createExportDownload("1", storageId);
var statuses = downloadClient.getDownloadStatus("1");
assertThat(statuses).isNotEmpty();
assertThat(statuses.get(0).getStatus()).isEqualTo(DownloadStatusValue.READY);
ImportDossierTemplateRequest request1 = ImportDossierTemplateRequest.builder()
.dossierTemplateId(dossierTemplate.getId())
.updateExistingDossierTemplate(true)
.userId("1")
.archive(storageService.getObject(storageId).getInputStream().readAllBytes())
.build();
dossierTemplateClient.importDossierTemplate(request1);
var updatedDossierTemplate = dossierTemplateClient.getDossierTemplate(dossierTemplate.getId());
assertThat(updatedDossierTemplate.getModifiedBy()).isEqualTo("1");
assertThat(updatedDossierTemplate.getDateModified()).isNotNull();
assertThat(watermarkClient.getWatermarksForDossierTemplateId(updatedDossierTemplate.getId()).size()).isEqualTo(1);
}
private void setupDossierTemplate(DossierTemplate dossierTemplate) {
Type type = Type.builder()
.type("t")
.dossierTemplateId(dossierTemplate.getId())
.hexColor("#ddddd")
.recommendationHexColor("#cccccc")
.skippedHexColor("#cccccc")
.rank(999)
.isHint(false)
.isRecommendation(false)
.isCaseInsensitive(false)
.description("t.getDescription()")
.addToDictionaryAction(true)
.label("t.getLabel()")
.dossierId(null)
.version(23)
.entries(List.of(DictionaryEntry.builder().entryId(1001).value("dictEntry1").version(23).deleted(false).typeId("t.getType()").build()))
.falsePositiveEntries(List.of(DictionaryEntry.builder().entryId(2001).value("dictEntry2").version(23).deleted(false).typeId("t.getType()").build()))
.falseRecommendationEntries(List.of(DictionaryEntry.builder().entryId(3001).value("dictEntry3").version(23).deleted(false).typeId("t.getType()").build()))
.hasDictionary(true)
.systemManaged(false)
.build();
Type type2 = Type.builder()
.type("t2")
.dossierTemplateId(dossierTemplate.getId())
.hexColor("#12345")
.recommendationHexColor("#6789a")
.skippedHexColor("#6789a")
.rank(1002)
.isHint(false)
.isRecommendation(false)
.isCaseInsensitive(false)
.description("t2.getDescription()")
.addToDictionaryAction(true)
.label("t2.getLabel()")
.dossierId(null)
.version(23)
.entries(List.of(DictionaryEntry.builder().entryId(1001).value("dictEntry1").version(23).deleted(false).typeId("t2.getType()").build()))
.falsePositiveEntries(List.of(DictionaryEntry.builder().entryId(2001).value("dictEntry2").version(23).deleted(false).typeId("t2.getType()").build()))
.falseRecommendationEntries(List.of(DictionaryEntry.builder().entryId(3001).value("dictEntry3").version(23).deleted(false).typeId("t2.getType()").build()))
.hasDictionary(true)
.systemManaged(false)
.build();
var createdType1 = dictionaryClient.addType(type);
var createdType2 = dictionaryClient.addType(type2);
var loadedType1 = dictionaryClient.getDictionaryForType(createdType1.getId(), null);
var loadedType2 = dictionaryClient.getDictionaryForType(createdType2.getId(), null);
dictionaryClient.addEntries(loadedType1.getTypeId(), List.of("entry1", "entry2"), false, false, DictionaryEntryType.ENTRY);
dictionaryClient.addEntries(loadedType2.getTypeId(), List.of("entry3", "entry4"), false, false, DictionaryEntryType.FALSE_POSITIVE);
dossierAttributeConfigClient.setDossierAttributesConfig(dossierTemplate.getId(), List.of(DossierAttributeConfig.builder()
.dossierTemplateId(dossierTemplate.getId())
.editable(false)
.id("dossierAttributeId")
.label("labelDossierAttribute")
.type(DossierAttributeType.TEXT)
.placeholder("placeholderDossier")
.build()));
fileAttributeConfigClient.setFileAttributesGeneralConfig(dossierTemplate.getId(), FileAttributesGeneralConfiguration.builder().dossierTemplateId(dossierTemplate.getId())
.delimiter("")
.filenameMappingColumnHeaderName("filenameMappingColumnHeaderName")
.build());
fileAttributeConfigClient.setFileAttributesConfig(dossierTemplate.getId(), List.of(FileAttributeConfig.builder()
.dossierTemplateId(dossierTemplate.getId())
.primaryAttribute(true)
.csvColumnHeader("12345")
.displayedInFileList(false)
.editable(false)
.filterable(false)
.id("fileAttributeId")
.label("labelFileAttribute")
.type(FileAttributeType.TEXT)
.placeholder("placeholderFile")
.build()));
ReportTemplateEntity rte = ReportTemplateEntity.builder()
.templateId("templateId")
.dossierTemplate(convert(dossierTemplate, DossierTemplateEntity.class))
.dossierTemplateId(dossierTemplate.getId())
.dossiers(null)
.activeByDefault(false)
.fileName("rte")
.multiFileReport(false)
.storageId("storageId")
.uploadDate(OffsetDateTime.now())
.build();
reportTemplateClient.uploadTemplate(ReportTemplateUploadRequest.builder()
.template("some text".getBytes(StandardCharsets.UTF_8))
.dossierTemplateId(dossierTemplate.getId())
.fileName("Report Template")
.activeByDefault(true)
.multiFileReport(false)
.build());
var col = Colors.builder()
.analysisColor("#111111")
.dossierTemplateId(dossierTemplate.getId())
.dictionaryRequestColor("#333333")
.requestAddColor("#444444")
.ignoredHintColor("#555555")
.requestRemoveColor("#666666")
.analysisColor("#777777")
.previewColor("#888888")
.redactionColor("#999999")
.updatedColor("#aaaaaa")
.build();
dictionaryClient.setColors(dossierTemplate.getId(), col);
var dossierStatus = CreateOrUpdateDossierStatusRequest.builder()
.name("dossStatus1")
.description("ds description")
.color("#115599")
.rank(456)
.dossierTemplateId(dossierTemplate.getId())
.build();
dossierStatusClient.createOrUpdateDossierStatus(dossierStatus);
Watermark watermark = new Watermark();
watermark.setName("watermark name");
watermark.setEnabled(true);
watermark.setText("Minions ipsum chasy para tu la bodaaa bananaaaa hana dul sae. Chasy hana dul sae pepete hana dul sae belloo! Tatata bala tu ti aamoo! Jeje.");
watermark.setFontSize(12);
watermark.setFontType("font");
watermark.setHexColor("#dddddd");
watermark.setOpacity(20);
watermark.setOrientation(WatermarkOrientation.DIAGONAL);
watermark.setDossierTemplateId(dossierTemplate.getId());
watermark.setCreatedBy("user");
watermarkClient.createOrUpdateWatermark(watermark);
}
}

4
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java
View File

@ -113,7 +113,9 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
// disable the watermark used
watermarkConfig.setEnabled(false);
watermarkClient.createOrUpdateWatermark(watermarkConfig);
var watermarkDisabled = watermarkClient.createOrUpdateWatermark(watermarkConfig);
assertThat(watermarkDisabled.isEnabled()).isFalse();
assertThat(watermarkDisabled.getId()).isEqualTo(watermarkConfig.getId());
// update dossier description, while the watermark used was disabled
cru.setDescription("new description");
updated = dossierClient.updateDossier(cru, dossier.getId());