diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts b/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts index 1cc96b271..fc349e6ef 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts @@ -7,6 +7,7 @@ dependencies { api(project(":persistence-service-processor-v1")) api(project(":persistence-service-external-api-v2")) api(project(":persistence-service-external-api-impl-v1")) + implementation("org.mapstruct:mapstruct:1.5.5.Final") annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java index a21eb22fd..943fba2d0 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java @@ -1,45 +1,25 @@ package com.iqser.red.persistence.service.v2.external.api.impl.controller; -import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_RULES; -import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.WRITE_RULES; import static com.iqser.red.service.persistence.service.v2.api.external.resource.DossierResource.DOSSIER_ID_PARAM; import static com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource.DOSSIER_TEMPLATE_ID_PARAM; import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_ID_PARAM; -import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.multipart.MultipartFile; -import com.google.common.base.Strings; import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController; -import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMappingMapper; -import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; -import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService; -import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntityReference; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue; import com.iqser.red.service.persistence.service.v2.api.external.model.Component; -import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; -import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentValue; import com.iqser.red.service.persistence.service.v2.api.external.model.EntityReference; import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents; @@ -49,7 +29,6 @@ import com.iqser.red.service.persistence.service.v2.api.external.resource.Compon import io.swagger.v3.oas.annotations.tags.Tag; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.experimental.FieldDefaults; @RestController @@ -61,8 +40,6 @@ public class ComponentControllerV2 implements ComponentResource { ComponentLogService componentLogService; StatusController statusController; FileStatusService fileStatusService; - ComponentMappingService componentMappingService; - ComponentMappingMapper componentMappingMapper = ComponentMappingMapper.INSTANCE; DossierTemplatePersistenceService dossierTemplatePersistenceService; @@ -150,101 +127,4 @@ public class ComponentControllerV2 implements ComponentResource { } - @Override - @PreAuthorize("hasAuthority('" + READ_RULES + "')") - public ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) { - - dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - List summaries = componentMappingService.getMetaDataByDossierTemplateId(dossierTemplateId); - List componentMappingMetadataModelList = componentMappingMapper.toModelList(summaries); - return new ComponentMappingSummary(dossierTemplateId, componentMappingMetadataModelList); - } - - - @Override - @SneakyThrows - @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") - public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, char delimiter) { - - dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - - String nameToUse = Strings.isNullOrEmpty(name) ? file.getName().split("\\.")[0] : name; - - if (Strings.isNullOrEmpty(nameToUse)) { - throw new BadRequestException("The provided file name is not valid!"); - } - - Path mappingFile = saveToFile(file); - String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); - - com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, delimiter, encoding, mappingFile.toFile()); - - Files.deleteIfExists(mappingFile); - - return componentMappingMapper.toModel(metaData); - } - - - @Override - @SneakyThrows - @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") - public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String encoding, char delimiter) { - - dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - - Path mappingFile = saveToFile(file); - - com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata resultMetaData = componentMappingService.update(componentMappingId, encoding, delimiter, mappingFile.toFile()); - - Files.deleteIfExists(mappingFile); - - return componentMappingMapper.toModel(resultMetaData); - } - - - @Override - @PreAuthorize("hasAuthority('" + READ_RULES + "')") - public ResponseEntity downloadMapping(String dossierTemplateId, String componentMappingId) { - - dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - - ComponentMappingDownloadModel mappingDownloadModel = componentMappingService.getMappingForDownload(componentMappingId); - - HttpHeaders httpHeaders = new HttpHeaders(); - httpHeaders.setContentType(MediaType.TEXT_PLAIN); - - httpHeaders.add("Content-Disposition", - "attachment" - + "; filename*=" - + mappingDownloadModel.encoding().toLowerCase(Locale.US) - + "''" - + StringEncodingUtils.urlEncode(mappingDownloadModel.fileName())); - - return new ResponseEntity<>(mappingDownloadModel.mappingFileResource(), httpHeaders, HttpStatus.OK); - } - - - @Override - @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") - public ResponseEntity deleteMapping(String dossierTemplateId, String componentMappingId) { - - dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - - componentMappingService.delete(componentMappingId); - - return new ResponseEntity<>(HttpStatus.NO_CONTENT); - } - - - @SneakyThrows - private static Path saveToFile(MultipartFile file) { - - Path mappingFile = Files.createTempFile(file.getName(), ".csv"); - - try (var out = new FileOutputStream(mappingFile.toFile())) { - out.write(file.getBytes()); - } - return mappingFile; - } - } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java index 77cb78cad..9aaf1eb7e 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java @@ -6,8 +6,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.WRITE_RULES; import java.io.ByteArrayInputStream; +import java.io.FileOutputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Locale; @@ -23,12 +26,17 @@ import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import com.google.common.base.Strings; import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierTemplateController; import com.iqser.red.persistence.service.v1.external.api.impl.controller.FileAttributesController; +import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMappingMapper; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.utils.RulesValidationMapper; import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils; @@ -38,6 +46,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinition; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList; import com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource; @@ -47,22 +57,27 @@ import feign.FeignException; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; @RestController @RequiredArgsConstructor @Tag(name = "1. Dossier templates endpoints", description = "Provides operations related to dossier templates") +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DossierTemplateControllerV2 implements DossierTemplateResource { private static final String RULES_DOWNLOAD_FILE_NAME_SUFFIX = "-rules.drl"; - private final DossierTemplateController dossierTemplateController; - private final RulesPersistenceService rulesPersistenceService; - private final RulesValidationService rulesValidationService; - private final AuditPersistenceService auditPersistenceService; - private final FileAttributesController fileAttributesController; - + DossierTemplateController dossierTemplateController; + RulesPersistenceService rulesPersistenceService; + RulesValidationService rulesValidationService; + AuditPersistenceService auditPersistenceService; + FileAttributesController fileAttributesController; + ComponentMappingService componentMappingService; + ComponentMappingMapper componentMappingMapper = ComponentMappingMapper.INSTANCE; + DossierTemplatePersistenceService dossierTemplatePersistenceService; public List getAllDossierTemplates() { @@ -212,4 +227,102 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { return new ResponseEntity<>(new InputStreamResource(is), httpHeaders, HttpStatus.OK); } + + @Override + @PreAuthorize("hasAuthority('" + READ_RULES + "')") + public ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + List summaries = componentMappingService.getMetaDataByDossierTemplateId(dossierTemplateId); + List componentMappingMetadataModelList = componentMappingMapper.toModelList(summaries); + return new ComponentMappingSummary(dossierTemplateId, componentMappingMetadataModelList); + } + + + @Override + @SneakyThrows + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, char delimiter) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + String nameToUse = Strings.isNullOrEmpty(name) ? file.getName().split("\\.")[0] : name; + + if (Strings.isNullOrEmpty(nameToUse)) { + throw new BadRequestException("The provided file name is not valid!"); + } + + Path mappingFile = saveToFile(file); + String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); + + com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, delimiter, encoding, mappingFile.toFile()); + + Files.deleteIfExists(mappingFile); + + return componentMappingMapper.toModel(metaData); + } + + + @Override + @SneakyThrows + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String encoding, char delimiter) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + Path mappingFile = saveToFile(file); + + com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata resultMetaData = componentMappingService.update(componentMappingId, encoding, delimiter, mappingFile.toFile()); + + Files.deleteIfExists(mappingFile); + + return componentMappingMapper.toModel(resultMetaData); + } + + + @Override + @PreAuthorize("hasAuthority('" + READ_RULES + "')") + public ResponseEntity downloadMapping(String dossierTemplateId, String componentMappingId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + ComponentMappingDownloadModel mappingDownloadModel = componentMappingService.getMappingForDownload(componentMappingId); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.TEXT_PLAIN); + + httpHeaders.add("Content-Disposition", + "attachment" + + "; filename*=" + + mappingDownloadModel.encoding().toLowerCase(Locale.US) + + "''" + + StringEncodingUtils.urlEncode(mappingDownloadModel.fileName())); + + return new ResponseEntity<>(mappingDownloadModel.mappingFileResource(), httpHeaders, HttpStatus.OK); + } + + + @Override + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ResponseEntity deleteMapping(String dossierTemplateId, String componentMappingId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + componentMappingService.delete(componentMappingId); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + + @SneakyThrows + private static Path saveToFile(MultipartFile file) { + + Path mappingFile = Files.createTempFile(file.getName(), ".csv"); + + try (var out = new FileOutputStream(mappingFile.toFile())) { + out.write(file.getBytes()); + } + return mappingFile; + } + } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java new file mode 100644 index 000000000..6276722e1 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java @@ -0,0 +1,184 @@ +package com.iqser.red.persistence.service.v2.external.api.impl.controller; + +import java.time.OffsetDateTime; +import java.util.Map; +import java.util.stream.Collectors; + +import org.quartz.JobDataMap; +import org.quartz.JobKey; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.HttpStatus; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.WebDataBinder; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.multipart.support.MissingServletRequestPartException; + +import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import com.iqser.red.commons.spring.ErrorMessage; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.knecon.fforesight.tenantcommons.TenantContext; +import com.mchange.rmi.NotAuthorizedException; + +import io.swagger.v3.oas.annotations.Hidden; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +@RestControllerAdvice +@Order(Ordered.HIGHEST_PRECEDENCE) +public class ExternalControllerAdviceV2 { + + private final Scheduler scheduler; + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.NOT_FOUND) + @ExceptionHandler(value = NotFoundException.class) + public ErrorMessage handleContentNotFoundException(NotFoundException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + /* error handling */ + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler(value = BadRequestException.class) + public ErrorMessage handleBadRequestException(BadRequestException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.CONFLICT) + @ExceptionHandler(value = {ConflictException.class}) + protected ErrorMessage handleConflictException(ConflictException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.FORBIDDEN) + @ExceptionHandler({AccessDeniedException.class}) + public ErrorMessage handleAccessDeniedException(AccessDeniedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.UNAUTHORIZED) + @ExceptionHandler({NotAuthorizedException.class}) + public ErrorMessage handleNotAuthorizedException(NotAuthorizedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.FORBIDDEN) + @ExceptionHandler({NotAllowedException.class}) + public ErrorMessage handleNotAllowedException(NotAllowedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + @ExceptionHandler({org.springframework.security.acls.model.NotFoundException.class}) + public ErrorMessage handleACLNotFound(org.springframework.security.acls.model.NotFoundException e) { + + // in case this error occurs on a rest request / force trigger the sync job + try { + scheduler.triggerJob(new JobKey("SyncUserPermissionsJob"), new JobDataMap(Map.of("tenantId", TenantContext.getTenantId()))); + } catch (SchedulerException ex) { + log.debug("Failed to force trigger SyncUserPermissionsJob", ex); + } + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler({MethodArgumentNotValidException.class}) + public ErrorMessage handleMethodArgumentNotValidException(MethodArgumentNotValidException e) { + + var errorList = e.getFieldErrors(); + String errorListAsString = errorList.stream() + .map(fieldError -> fieldError.getField() + ": " + fieldError.getDefaultMessage()) + .collect(Collectors.joining(", ")); + return new ErrorMessage(OffsetDateTime.now(), String.format("You have empty/wrong formatted parameters: %s", errorListAsString)); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler({MissingServletRequestPartException.class}) + public ErrorMessage handleMissingServletRequestPartException(MissingServletRequestPartException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler({HttpMessageNotReadableException.class}) + public ErrorMessage handleHttpMessageNotReadableException(HttpMessageNotReadableException e) { + + var cause = e.getCause(); + if (cause instanceof InvalidFormatException) { + InvalidFormatException invalidFormatException = (InvalidFormatException) cause; + + Class targetType = invalidFormatException.getTargetType(); + if (targetType != null && targetType.isEnum()) { + return new ErrorMessage(OffsetDateTime.now(), String.format("Unsupported value for %s", targetType.getSimpleName())); + } + + return new ErrorMessage(OffsetDateTime.now(), cause.getMessage()); + } + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Order(10000) + public static class BinderControllerAdvice { + + @InitBinder + public void setAllowedFields(WebDataBinder dataBinder) { + // This code protects Spring Core from a "Remote Code Execution" attack (dubbed "Spring4Shell"). + // By applying this mitigation, you prevent the "Class Loader Manipulation" attack vector from firing. + // For more details, see this post: https://www.lunasec.io/docs/blog/spring-rce-vulnerabilities/ + String[] denylist = new String[]{"class.*", "Class.*", "*.class.*", "*.Class.*"}; + dataBinder.setDisallowedFields(denylist); + } + + } + +} diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java index c2c7aef7d..a67b78e9c 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java @@ -49,19 +49,10 @@ public interface ComponentResource { String INCLUDE_DETAILS_DESCRIPTION = """ A toggle to decide whether to include detailed component information in the response: - - true: The component object's field componentDetails stores detailed information about the source of its respective value(s). - false (default): The component object does not contain a field componentDetails. """; - String COMPONENT_MAPPINGS_PATH = COMPONENTS_PATH + "/mappings"; - String COMPONENT_MAPPING_ID_PARAM = "componentMappingId"; - String COMPONENT_MAPPING_ID_PATH_VARIABLE = "/{" + COMPONENT_MAPPING_ID_PARAM + "}"; - - String RULE_FILE_TYPE_PARAMETER_NAME = "ruleFileType"; - String ENCODING_PARAM = "encoding"; - String DELIMITER_PARAM = "delimiter"; - String MAPPING_NAME_PARAM = "name"; @GetMapping(value = FILE_PATH + FILE_ID_PATH_VARIABLE + COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) @@ -81,50 +72,5 @@ public interface ComponentResource { @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails); - @Operation(summary = "Get the component mapping summaries of a DossierTemplate.", description = "None") - @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH, produces = MediaType.APPLICATION_JSON_VALUE) - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping views returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) - ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId); - - - @Operation(summary = "Upload a new component mapping to a DossierTemplate.", description = "None") - @PostMapping(value = PATH - + DOSSIER_TEMPLATE_ID_PATH_VARIABLE - + COMPONENT_MAPPINGS_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping uploaded successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) - ComponentMappingMetadataModel uploadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, - @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, - @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name, - @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, - @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); - - - @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None") - @PutMapping(value = PATH - + DOSSIER_TEMPLATE_ID_PATH_VARIABLE - + COMPONENT_MAPPINGS_PATH - + COMPONENT_MAPPING_ID_PATH_VARIABLE, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping updated successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) - ComponentMappingMetadataModel updateMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, - @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId, - @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, - @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, - @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); - - - @ResponseBody - @ResponseStatus(value = HttpStatus.OK) - @Operation(summary = "Returns file containing the specified mapping as a file.") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) - @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE, produces = MediaType.MULTIPART_FORM_DATA_VALUE) - ResponseEntity downloadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId); - - - @ResponseBody - @ResponseStatus(value = HttpStatus.OK) - @Operation(summary = "Deletes a specified mapping.") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) - @DeleteMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE, produces = MediaType.MULTIPART_FORM_DATA_VALUE) - ResponseEntity deleteMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId); } diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java index 116cb7717..9f7046383 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java @@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.service.v2.api.external.resource; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList; import io.swagger.v3.oas.annotations.Operation; @@ -13,9 +15,11 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.ResponseBody; @@ -33,17 +37,20 @@ public interface DossierTemplateResource { String ENTITY_RULES_PATH = "/entity-rules"; String COMPONENT_RULES_PATH = "/component-rules"; - - + String COMPONENT_MAPPINGS_PATH = "/component-mappings"; String FILE_ATTRIBUTE_DEFINITIONS_PATH = "/file-attribute-definitions"; String DOSSIER_TEMPLATE_ID_PARAM = "dossierTemplateId"; String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID_PARAM + "}"; + String COMPONENT_MAPPING_ID_PARAM = "componentMappingId"; + String COMPONENT_MAPPING_ID_PATH_VARIABLE = "/{" + COMPONENT_MAPPING_ID_PARAM + "}"; + String DRY_RUN_PARAM = "dryRun"; - - + String ENCODING_PARAM = "encoding"; + String DELIMITER_PARAM = "delimiter"; + String MAPPING_NAME_PARAM = "name"; @GetMapping(value = PATH, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Lists all existing DossierTemplates.", description = "None") @@ -98,4 +105,51 @@ public interface DossierTemplateResource { @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "File attribute definitions returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) FileAttributeDefinitionList getFileAttributeDefinitions(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId); + + @Operation(summary = "Get the component mapping summaries of a DossierTemplate.", description = "None") + @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping views returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) + ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId); + + + @Operation(summary = "Upload a new component mapping to a DossierTemplate.", description = "None") + @PostMapping(value = PATH + + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + + COMPONENT_MAPPINGS_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping uploaded successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + ComponentMappingMetadataModel uploadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, + @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name, + @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, + @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); + + + @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None") + @PutMapping(value = PATH + + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + + COMPONENT_MAPPINGS_PATH + + COMPONENT_MAPPING_ID_PATH_VARIABLE, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping updated successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + ComponentMappingMetadataModel updateMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, + @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId, + @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, + @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); + + + @ResponseBody + @ResponseStatus(value = HttpStatus.OK) + @Operation(summary = "Returns file containing the specified mapping as a file.") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) + @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE, produces = MediaType.MULTIPART_FORM_DATA_VALUE) + ResponseEntity downloadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId); + + + @ResponseBody + @ResponseStatus(value = HttpStatus.OK) + @Operation(summary = "Deletes a specified mapping.") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + @DeleteMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE, produces = MediaType.MULTIPART_FORM_DATA_VALUE) + ResponseEntity deleteMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId); + } diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml index baf9fe7b9..e715f335b 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml @@ -290,6 +290,179 @@ paths: $ref: '#/components/schemas/FileAttributeDefinitionList' description: | Successfully returned the file attribute definitions for the specified dossier template. + /api/dossier-templates/{dossierTemplateId}/component-mappings: + get: + operationId: listAllMappings + tags: + - 1. Dossier Templates + summary: Returns a list of all existing component mappings in a dossier template + description: | + Retrieves a collection of component mapping views associated with a specific dossier template. Each component mapping view includes details such as name, number of lines, delimiter, encoding and other relevant metadata. This endpoint is useful for clients needing to understand what mappings are available under a particular dossier template. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMappingSummary' + description: | + Successfully returned the component mapping summary for the specified dossier template. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + post: + operationId: uploadMapping + summary: Upload a new component mapping to a DossierTemplate. + description: | + Utilize this endpoint to upload a new component mapping to a designated DossierTemplate. + The file is expected to be a comma separated file, whose first row are the labels for the columns of the file. + Further, it is expected that the data is rectangular, this means each row has the same size. + The rows in the file will be sorted the values in each column, starting with the left most and moving right recursively. + This enables much faster lookups down the line. + This means, it is highly beneficial to structure the CSV File such that the keys to query for appear in the first rows and the results to map in the last. + tags: + - 1. Dossier Templates + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/mappingName' + - $ref: '#/components/parameters/encoding' + - $ref: '#/components/parameters/delimiter' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMappingMetadata' + description: | + Component mapping uploaded successfully and returned the component mapping metadata. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + + /api/dossier-templates/{dossierTemplateId}/component-mappings/{comonentMappingId}: + get: + operationId: downloadMappingFile + tags: + - 1. Dossier Templates + summary: Download a specific component mapping file of a specific dossier template. + description: | + Utilize this endpoint to download a specific component mapping file of a designated dossier template. + The file is named the same and encoded the same as when it was uploaded, but it's sorting might have changed to provide faster lookups. + A component mapping file may be used in the component rules to relate components to existing master data. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + responses: + "200": + headers: + Content-Disposition: + schema: + type: string + example: attachment; filename*=utf-8''mapping.csv + content: + text/plain; charset=utf-8: + schema: + type: string + description: | + Successfully downloaded the requested component mapping. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + put: + operationId: updateMapping + summary: Update an existing component mapping of a DossierTemplate. + description: Updates an existing component mapping, + tags: + - 1. Dossier Templates + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + - $ref: '#/components/parameters/encoding' + - $ref: '#/components/parameters/delimiter' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMappingMetadata' + description: | + Component mapping uploaded successfully and returned the component mapping metadata. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + delete: + operationId: deleteMappingFile + tags: + - 1. Dossier Templates + summary: Delete a specific component mapping file of a specific dossier template. + description: | + Utilize this endpoint to delete a specific component mapping file of a designated dossier template. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + responses: + "204": + description: | + Successfully deleted the requested component mapping. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' /api/dossier-templates/{dossierTemplateId}/dossiers: get: operationId: getDossiers @@ -304,7 +477,7 @@ paths: Use this endpoint to fetch the required dossiers before performing actions on specific ones. Use the query parameters to modify the response. E.g., set the `includeArchivedDossiers` parameter to `true` so that the response also contains - *archived* dossiers. + *archived* dossiers. parameters: - $ref: '#/components/parameters/dossierTemplateId' - $ref: '#/components/parameters/includeActiveDossiers' @@ -745,179 +918,6 @@ paths: $ref: '#/components/responses/403' "500": $ref: '#/components/responses/500' - /api/dossier-templates/{dossierTemplateId}/component-mappings: - get: - operationId: listAllMappings - tags: - - 4. Components - summary: Returns a list of all existing component mappings in a dossier template - description: | - Retrieves a collection of component mapping views associated with a specific dossier template. Each component mapping view includes details such as name, number of lines, delimiter, encoding and other relevant metadata. This endpoint is useful for clients needing to understand what mappings are available under a particular dossier template. - parameters: - - $ref: '#/components/parameters/dossierTemplateId' - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/ComponentMappingSummary' - description: | - Successfully returned the component mapping summary for the specified dossier template. - "400": - $ref: '#/components/responses/400' - "401": - $ref: '#/components/responses/401' - "403": - $ref: '#/components/responses/403' - "404": - $ref: '#/components/responses/404-dossier-template' - "429": - $ref: '#/components/responses/429' - "500": - $ref: '#/components/responses/500' - post: - operationId: uploadMapping - summary: Upload a new component mapping to a DossierTemplate. - description: | - Utilize this endpoint to upload a new component mapping to a designated DossierTemplate. - The file is expected to be a comma separated file, whose first row are the labels for the columns of the file. - Further, it is expected that the data is rectangular, this means each row has the same size. - The rows in the file will be sorted the values in each column, starting with the left most and moving right recursively. - This enables much faster lookups down the line. - This means, it is highly beneficial to structure the CSV File such that the keys to query for appear in the first rows and the results to map in the last. - tags: - - 4. Components - requestBody: - content: - multipart/form-data: - schema: - $ref: '#/components/schemas/UploadRequest' - parameters: - - $ref: '#/components/parameters/dossierTemplateId' - - $ref: '#/components/parameters/mappingName' - - $ref: '#/components/parameters/encoding' - - $ref: '#/components/parameters/delimiter' - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/ComponentMappingMetadata' - description: | - Component mapping uploaded successfully and returned the component mapping metadata. - "400": - $ref: '#/components/responses/400' - "401": - $ref: '#/components/responses/401' - "403": - $ref: '#/components/responses/403' - "404": - $ref: '#/components/responses/404-dossier-template' - "429": - $ref: '#/components/responses/429' - "500": - $ref: '#/components/responses/500' - - /api/dossier-templates/{dossierTemplateId}/component-mappings/{comonentMappingId}: - get: - operationId: downloadMappingFile - tags: - - 4. Components - summary: Download a specific component mapping file of a specific dossier template. - description: | - Utilize this endpoint to download a specific component mapping file of a designated dossier template. - The file is named the same and encoded the same as when it was uploaded, but it's sorting might have changed to provide faster lookups. - A component mapping file may be used in the component rules to relate components to existing master data. - parameters: - - $ref: '#/components/parameters/dossierTemplateId' - - $ref: '#/components/parameters/componentMappingId' - responses: - "200": - headers: - Content-Disposition: - schema: - type: string - example: attachment; filename*=utf-8''mapping.csv - content: - text/plain; charset=utf-8: - schema: - type: string - description: | - Successfully downloaded the requested component mapping. - "400": - $ref: '#/components/responses/400' - "401": - $ref: '#/components/responses/401' - "403": - $ref: '#/components/responses/403' - "404": - $ref: '#/components/responses/404-dossier-template' - "429": - $ref: '#/components/responses/429' - "500": - $ref: '#/components/responses/500' - put: - operationId: updateMapping - summary: Update an existing component mapping of a DossierTemplate. - description: Updates an existing component mapping, - tags: - - 4. Components - requestBody: - content: - multipart/form-data: - schema: - $ref: '#/components/schemas/UploadRequest' - parameters: - - $ref: '#/components/parameters/dossierTemplateId' - - $ref: '#/components/parameters/componentMappingId' - - $ref: '#/components/parameters/encoding' - - $ref: '#/components/parameters/delimiter' - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/ComponentMappingMetadata' - description: | - Component mapping uploaded successfully and returned the component mapping metadata. - "400": - $ref: '#/components/responses/400' - "401": - $ref: '#/components/responses/401' - "403": - $ref: '#/components/responses/403' - "404": - $ref: '#/components/responses/404-dossier-template' - "429": - $ref: '#/components/responses/429' - "500": - $ref: '#/components/responses/500' - delete: - operationId: deleteMappingFile - tags: - - 4. Components - summary: Delete a specific component mapping file of a specific dossier template. - description: | - Utilize this endpoint to delete a specific component mapping file of a designated dossier template. - parameters: - - $ref: '#/components/parameters/dossierTemplateId' - - $ref: '#/components/parameters/componentMappingId' - responses: - "204": - description: | - Successfully deleted the requested component mapping. - "400": - $ref: '#/components/responses/400' - "401": - $ref: '#/components/responses/401' - "403": - $ref: '#/components/responses/403' - "404": - $ref: '#/components/responses/404-dossier-template' - "429": - $ref: '#/components/responses/429' - "500": - $ref: '#/components/responses/500' /api/license/active/usage: post: operationId: getReport @@ -1093,7 +1093,7 @@ components: minLength: 1 maxLength: 1 example: ',' - description: "The delimiter used in a csv file. Default is ','." + description: "The delimiter used as a separator in a csv file. Default is ','." mappingName: name: name required: false diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java index 25ebd46f8..bfbd4642b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java @@ -87,6 +87,9 @@ public class ComponentMappingPersistenceService { public ComponentMappingDownloadModel getMappingFileForDownload(String componentMappingId) { var entity = getEntityById(componentMappingId); + if (!storageService.objectExists(TenantContext.getTenantId(), entity.getStorageId())) { + throw new NotFoundException("ComponentMapping with id " + componentMappingId + " does not exist!"); + } return new ComponentMappingDownloadModel(storageService.getObject(TenantContext.getTenantId(), entity.getStorageId()), entity.getEncoding(), entity.getFileName()); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java index 751fa1f91..4ea8c31fe 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java @@ -140,7 +140,7 @@ public class ComponentMappingService { columnLabels = rows.remove(0); - numberOfLines = (int) reader.getLinesRead(); + numberOfLines = (int) reader.getLinesRead() - 1; rows.sort(CSV_SORTER); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java index e9bc1c8cc..f09b35fb7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -10,6 +12,7 @@ import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; +import org.springframework.util.FileSystemUtils; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity; @@ -20,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService; @@ -40,29 +44,33 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; import com.knecon.fforesight.tenantcommons.TenantContext; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DossierTemplateCloneService { - private final DossierTemplateRepository dossierTemplateRepository; - private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; - private final RulesPersistenceService rulesPersistenceService; - private final DossierTemplatePersistenceService dossierTemplatePersistenceService; - private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService; - private final DictionaryPersistenceService dictionaryPersistenceService; - private final EntryPersistenceService entryPersistenceService; - private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; - private final ReportTemplatePersistenceService reportTemplatePersistenceService; - private final ColorsService colorsService; - private final StorageService storageService; - private final DossierStatusPersistenceService dossierStatusPersistenceService; - private final WatermarkService watermarkService; - private final FileManagementStorageService fileManagementStorageService; + DossierTemplateRepository dossierTemplateRepository; + LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + RulesPersistenceService rulesPersistenceService; + DossierTemplatePersistenceService dossierTemplatePersistenceService; + DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService; + DictionaryPersistenceService dictionaryPersistenceService; + EntryPersistenceService entryPersistenceService; + FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; + ReportTemplatePersistenceService reportTemplatePersistenceService; + ColorsService colorsService; + StorageService storageService; + DossierStatusPersistenceService dossierStatusPersistenceService; + WatermarkService watermarkService; + FileManagementStorageService fileManagementStorageService; + ComponentMappingService componentMappingService; public DossierTemplateEntity cloneDossierTemplate(String dossierTemplateId, CloneDossierTemplateRequest cloneDossierTemplateRequest) { @@ -129,6 +137,8 @@ public class DossierTemplateCloneService { // set the watermark configurations cloneWatermarks(dossierTemplateId, clonedDossierTemplate.getId()); + cloneComponentMappings(dossierTemplateId, clonedDossierTemplate.getId()); + return clonedDossierTemplate; } @@ -153,6 +163,23 @@ public class DossierTemplateCloneService { } + @SneakyThrows + private void cloneComponentMappings(String dossierTemplateId, String clonedDossierTemplateId) { + + Path dir = Files.createTempDirectory("componentMappingsForClone"); + List componentMappings = componentMappingService.getMappingFilesByDossierTemplateId(dossierTemplateId, dir); + for (ComponentMapping componentMapping : componentMappings) { + componentMappingService.create(clonedDossierTemplateId, + componentMapping.metaData().getName(), + componentMapping.metaData().getFileName(), + componentMapping.metaData().getDelimiter(), + componentMapping.metaData().getEncoding(), + componentMapping.file()); + } + FileSystemUtils.deleteRecursively(dir); + } + + private void cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) { var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java index d859d4468..30fd04279 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java @@ -8,9 +8,12 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -29,6 +32,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.compress.utils.FileNameUtils; +import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -72,6 +76,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ComponentMappingImportModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ExportFilename; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportTemplateResult; @@ -83,6 +88,7 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -148,8 +154,8 @@ public class DossierTemplateImportService { Map> typeEntriesMap = new HashMap<>(); Map> typeFalsePositivesMap = new HashMap<>(); Map> typeFalseRecommendationsMap = new HashMap<>(); - List componentMappingMetaData = new LinkedList<>(); - + Map mappingDataMap = new HashMap<>(); + List mappingMetadataList = new LinkedList<>(); while ((ze = zis.getNextZipEntry()) != null) { log.debug("---> " + ze.getName() + " ---- " + ze.isDirectory()); totalEntryArchive++; @@ -278,8 +284,15 @@ public class DossierTemplateImportService { reportTemplateFilenameList = reportTemplateList.stream() .map(rt -> rt.isMultiFileReport() ? rt.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : rt.getFileName()) .toList(); - } else if (ze.getName().contains(ExportFilename.COMPONENT_MAPPINGS.getFilename()) ) { - log.info(ze.getName()); + } else if (ze.getName().startsWith(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename())) { + if (ze.getName().contains(ExportFilename.COMPONENT_MAPPINGS_FILE.getFilename())) { + mappingMetadataList = objectMapper.readValue(bytes, new TypeReference<>() { + }); + } else if (ze.getName().endsWith(".csv")) { + String fileName = ze.getName().replace(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename() + "/", ""); + mappingDataMap.put(fileName, bytes); + } + } else { reportTemplateBytesMap.put(ze.getName(), bos); } @@ -305,6 +318,13 @@ public class DossierTemplateImportService { } } + + for (var metadata : mappingMetadataList) { + String fileName = metadata.getName() + ".csv"; + if (mappingDataMap.containsKey(fileName)) { + importTemplateResult.componentMappings.add(new ComponentMappingImportModel(metadata, mappingDataMap.get(fileName))); + } + } if (importTemplateResult.getDossierTemplate() == null) { throw new BadRequestException("Provided archive is faulty"); } @@ -342,9 +362,6 @@ public class DossierTemplateImportService { updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId()); dossierTemplateRepository.save(existingDossierTemplate); - // set rules - setRulesWhenCompiled(request, dossierTemplateId); - existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(existingDossierTemplate) .name())); @@ -455,12 +472,11 @@ public class DossierTemplateImportService { dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); dossierTemplateEntity.setCreatedBy(request.getUserId()); //set rules - setRulesWhenCompiled(request, dossierTemplateEntity.getId()); var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity); loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate)); - dossierTemplateId = loadedDossierTemplate.getId(); + dossierTemplateId = loadedDossierTemplate.getId(); // set colors this.setColors(dossierTemplateId, request.getColors()); @@ -534,12 +550,38 @@ public class DossierTemplateImportService { FileAttributesGeneralConfigurationEntity.class)); } + setRulesWhenCompiled(request, dossierTemplateId); + setComponentMappings(dossierTemplateId, request.getComponentMappings()); + long elapsedTime = System.currentTimeMillis() - start; log.info("stop import dossier template elapsedTime: " + elapsedTime + "for: " + dossierTemplateId); return dossierTemplateId; } + @SneakyThrows + private void setComponentMappings(String dossierTemplateId, List componentMappings) { + + List existingMappings = componentMappingService.getMetaDataByDossierTemplateId(dossierTemplateId); + existingMappings.forEach(metadata -> componentMappingService.delete(metadata.getId())); + + for (ComponentMappingImportModel componentMapping : componentMappings) { + File tmpFile = Files.createTempFile("mapping", ".csv").toFile(); + try (var out = new FileOutputStream(tmpFile)) { + out.write(componentMapping.csvData()); + } + componentMappingService.create(dossierTemplateId, + componentMapping.metadata().getName(), + componentMapping.metadata().getFileName(), + componentMapping.metadata().getDelimiter(), + componentMapping.metadata().getEncoding(), + tmpFile); + assert tmpFile.delete(); + } + + } + + private void setRulesWhenCompiled(ImportTemplateResult request, String dossierTemplateId) { DroolsValidation droolsValidation = rulesValidationService.validateRules(RuleFileType.ENTITY, request.getRuleSet()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java index a60f18cd5..ba3903fa3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java @@ -89,7 +89,7 @@ public class DownloadService { request.getReportTemplateIds(), request.getRedactionPreviewColor()); websocketService.sendDownloadEvent(storageId, request.getUserId(), DownloadStatusValue.QUEUED); - addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).includeUnprocessed(request.getIncludeUnprocessed()).build(), 1); + addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).build(), 1); return new JSONPrimitive<>(storageId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java index daf20b270..03d332582 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java @@ -4,6 +4,7 @@ import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicC import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStreamWriter; @@ -48,6 +49,7 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.FileAttributesGeneralConfiguration; @@ -123,6 +125,7 @@ public class DossierTemplateExportService { objectMapper.registerModule(new JavaTimeModule()); DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId()); + downloadStatusPersistenceService.updateStatus(downloadJob.getStorageId(), DownloadStatusValue.GENERATING); String dossierTemplateId = extractDossierTemplateId(downloadStatus.getFilename()); var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId); @@ -244,18 +247,18 @@ public class DossierTemplateExportService { List componentMappings = componentMappingService.getMappingFilesByDossierTemplateId(dossierTemplateId, mappingDir); + List metadata = componentMappings.stream() + .map(ComponentMapping::metaData) + .toList(); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename(), + ExportFilename.COMPONENT_MAPPINGS_FILE.getFilename() + JSON_EXT, + objectMapper.writeValueAsBytes(metadata))); for (ComponentMapping componentMapping : componentMappings) { - - Path mappingFilePath = mappingDir.resolve(componentMapping.metaData().getFileName()); - - fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS.getFilename(), - componentMapping.metaData().getName() + JSON_EXT, - objectMapper.writeValueAsBytes(componentMapping))); - - fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS.getFilename(), - componentMapping.metaData().getName() + "-" + componentMapping.metaData() - .getFileName(), - Files.readAllBytes(mappingFilePath))); + try (var in = new FileInputStream(componentMapping.file())) { + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename(), + componentMapping.metaData().getName() + ".csv", + in.readAllBytes())); + } } FileSystemUtils.deleteRecursively(mappingDir); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java index a331e7341..19d4f8ced 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java @@ -128,7 +128,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { storageService, dossierStatusPersistenceService, watermarkService, - fileManagementStorageService); + fileManagementStorageService, + componentMappingService); dossierTemplateExportService = new DossierTemplateExportService(dossierTemplatePersistenceService, downloadStatusPersistenceService, dossierAttributeConfigPersistenceService, diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java new file mode 100644 index 000000000..0178bee6c --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java @@ -0,0 +1,189 @@ +package com.iqser.red.service.peristence.v1.server.integration.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.util.FileSystemUtils; + +import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; +import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService; +import com.iqser.red.service.persistence.management.v1.processor.service.export.DossierTemplateExportService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.SneakyThrows; + +public class DossierTemplateImportExportTest extends AbstractPersistenceServerServiceTest { + + public static final String IMPORTED_TEMPLATE_NAME = "imported-template"; + public static final String USER_ID = "Deine Mutter"; + public static final String AFTER = "after"; + public static final String BEFORE = "before"; + + @Autowired + DossierTemplateImportService dossierTemplateImportService; + + @Autowired + DossierTemplateExportService dossierTemplateExportService; + + @Autowired + StorageService storageService; + + @MockBean + DownloadStatusPersistenceService downloadStatusPersistenceService; + + + @AfterEach + public void clearStorage() { + + ((FileSystemBackedStorageService) storageService).clearStorage(); + } + + + @Test + @SneakyThrows + public void testImportExportRoundtrip() { + + TenantContext.setTenantId("redaction"); + Path outDir = Files.createTempDirectory(IMPORTED_TEMPLATE_NAME); + + Path dossierTemplateExportArchive = new ClassPathResource("files/dossiertemplates/DossierTemplate.zip").getFile().toPath(); + + String importedDossierTemplateId = dossierTemplateImportService.importDossierTemplate(ImportDossierTemplateRequest.builder() + .archive(Files.readAllBytes(dossierTemplateExportArchive)) + .userId(USER_ID) + .build()); + + when(downloadStatusPersistenceService.getStatus(anyString()))// + .thenReturn(DownloadStatusEntity.builder().filename(importedDossierTemplateId + ".zip").storageId(IMPORTED_TEMPLATE_NAME).build()); + + dossierTemplateExportService.createDownloadArchive(DownloadJob.builder().storageId(importedDossierTemplateId).userId(USER_ID).build()); + + File tmpFile = Files.createTempFile(IMPORTED_TEMPLATE_NAME, ".zip").toFile(); + storageService.downloadTo(TenantContext.getTenantId(), IMPORTED_TEMPLATE_NAME, tmpFile); + + unzip(dossierTemplateExportArchive.toFile().toString(), outDir.resolve(BEFORE)); + unzip(tmpFile.toString(), outDir.resolve(AFTER)); + + assert tmpFile.delete(); + + Map beforeContents = getDirectoryContents(outDir.resolve(BEFORE)); + Map afterContents = getDirectoryContents(outDir.resolve(AFTER)); + + assertEquals(beforeContents.size(), afterContents.size()); + assertEquals(beforeContents.keySet(), afterContents.keySet()); + + // can't assert equality on contents as UUID's are supposed to change on import + + FileSystemUtils.deleteRecursively(outDir); + } + + + public static void unzip(String zipFilePath, Path destDirectory) throws IOException { + + File destDir = destDirectory.toFile(); + if (!destDir.exists()) { + destDir.mkdir(); + } + try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath))) { + ZipEntry entry = zipIn.getNextEntry(); + while (entry != null) { + String filePath = destDirectory + File.separator + entry.getName(); + if (!entry.isDirectory()) { + // if the entry is a file, extracts it + extractFile(zipIn, filePath); + } else { + // if the entry is a directory, make the directory + File dir = new File(filePath); + dir.mkdirs(); + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); + } + } + } + + + private static void extractFile(ZipInputStream zipIn, String filePath) throws IOException { + + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) { + byte[] bytesIn = new byte[4096]; + int read; + while ((read = zipIn.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + } + } + + + private static boolean areDirectoriesIdentical(Path dir1, Path dir2) throws IOException, NoSuchAlgorithmException { + + if (!Files.isDirectory(dir1) || !Files.isDirectory(dir2)) { + throw new IllegalArgumentException("Both paths should be directories."); + } + + Map dir1Contents = getDirectoryContents(dir1); + Map dir2Contents = getDirectoryContents(dir2); + + return dir1Contents.equals(dir2Contents); + } + + + private static Map getDirectoryContents(Path dir) throws IOException, NoSuchAlgorithmException { + + Map contents = new HashMap<>(); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + + Files.walkFileTree(dir, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + + contents.put(dir.relativize(file), digest.digest(Files.readAllBytes(file))); + return FileVisitResult.CONTINUE; + } + + + @Override + public FileVisitResult preVisitDirectory(Path subDir, BasicFileAttributes attrs) throws IOException { + + contents.put(dir.relativize(subDir), new byte[0]); + return FileVisitResult.CONTINUE; + } + }); + + return contents; + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip new file mode 100644 index 000000000..bc076a829 Binary files /dev/null and b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip differ diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java index a2125b0c2..80112f08f 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java @@ -19,6 +19,5 @@ public enum ProcessingStatus { PRE_PROCESSED, FIGURE_DETECTION_ANALYZING, TABLE_PARSING_ANALYZING, - VISUAL_LAYOUT_PARSING_ANALYZING } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java new file mode 100644 index 000000000..1aa95b411 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport; + +import java.io.File; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; + +public record ComponentMappingImportModel(ComponentMappingMetadata metadata, byte[] csvData) { + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java index 82fe6b4a2..905b4fed0 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java @@ -20,7 +20,8 @@ public enum ExportFilename { ENTRIES("entries"), FALSE_POSITIVES("falsePositives"), FALSE_RECOMMENDATION("falseRecommendations"), - COMPONENT_MAPPINGS("componentMappings"); + COMPONENT_MAPPINGS_FOLDER("mappings"), + COMPONENT_MAPPINGS_FILE("componentMappingsList"); @Getter private final String filename; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java index fe4a7e29a..6f20e791e 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java @@ -50,6 +50,9 @@ public class ImportTemplateResult { @Builder.Default public List fileAttributesConfigs = new ArrayList<>(); + @Builder.Default + public List componentMappings = new ArrayList<>(); + @Builder.Default public List legalBases = new ArrayList<>();