diff --git a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts index c6b715907..eeff524f4 100644 --- a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts @@ -65,6 +65,15 @@ java { } allprojects { + + tasks.withType { + options { + this as StandardJavadocDocletOptions + addBooleanOption("Xdoclint:none", true) + addStringOption("Xmaxwarns", "1") + } + } + publishing { publications { create(name) { diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts b/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts index 88086af39..fc349e6ef 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts @@ -7,6 +7,9 @@ dependencies { api(project(":persistence-service-processor-v1")) api(project(":persistence-service-external-api-v2")) api(project(":persistence-service-external-api-impl-v1")) + + implementation("org.mapstruct:mapstruct:1.5.5.Final") + annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") } description = "persistence-service-external-api-impl-v2" diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java index 097de9abb..943fba2d0 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java @@ -12,10 +12,10 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierTemplateController; import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController; import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntityReference; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue; @@ -37,10 +37,10 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ComponentControllerV2 implements ComponentResource { - DossierTemplateController dossierTemplateController; ComponentLogService componentLogService; StatusController statusController; FileStatusService fileStatusService; + DossierTemplatePersistenceService dossierTemplatePersistenceService; @Override @@ -49,7 +49,7 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(FILE_ID_PARAM) String fileId, @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) { - dossierTemplateController.getDossierTemplate(dossierTemplateId); + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); var componentLog = componentLogService.getComponentLog(dossierId, fileId, true); Map> basicComponent = new LinkedHashMap<>(); @@ -119,11 +119,12 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) { - dossierTemplateController.getDossierTemplate(dossierTemplateId); + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); var dossierFiles = statusController.getDossierStatus(dossierId); return new FileComponentsList(dossierFiles.stream() .map(file -> getComponents(dossierTemplateId, dossierId, file.getFileId(), includeDetails)) .toList()); } + } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java index 2787e550f..3662294a1 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java @@ -6,8 +6,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.WRITE_RULES; import java.io.ByteArrayInputStream; +import java.io.FileOutputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Locale; @@ -23,11 +26,17 @@ import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import com.google.common.base.Strings; import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierTemplateController; import com.iqser.red.persistence.service.v1.external.api.impl.controller.FileAttributesController; +import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMappingMapper; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.utils.RulesValidationMapper; import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils; @@ -37,6 +46,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinition; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList; import com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource; @@ -46,21 +57,27 @@ import feign.FeignException; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; @RestController @RequiredArgsConstructor @Tag(name = "1. Dossier templates endpoints", description = "Provides operations related to dossier templates") +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DossierTemplateControllerV2 implements DossierTemplateResource { private static final String RULES_DOWNLOAD_FILE_NAME_SUFFIX = "-rules.drl"; - private final DossierTemplateController dossierTemplateController; - private final RulesPersistenceService rulesPersistenceService; - private final RulesValidationService rulesValidationService; - private final AuditPersistenceService auditPersistenceService; - private final FileAttributesController fileAttributesController; + DossierTemplateController dossierTemplateController; + RulesPersistenceService rulesPersistenceService; + RulesValidationService rulesValidationService; + AuditPersistenceService auditPersistenceService; + FileAttributesController fileAttributesController; + ComponentMappingService componentMappingService; + ComponentMappingMapper componentMappingMapper = ComponentMappingMapper.INSTANCE; + DossierTemplatePersistenceService dossierTemplatePersistenceService; public List getAllDossierTemplates() { @@ -162,6 +179,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { .dossierTemplateId(dossierTemplateId) .ruleFileType(ruleFileType) .build(); + DroolsValidationResponse rulesValidationResponse = new DroolsValidationResponse(); try { @@ -194,7 +212,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { private ResponseEntity downloadRules(String dossierTemplateId, RuleFileType ruleFileType) { - var ruleEntity = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType); + RuleSetEntity ruleEntity = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType); var data = ruleEntity.getValue().getBytes(StandardCharsets.UTF_8); @@ -209,4 +227,112 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { return new ResponseEntity<>(new InputStreamResource(is), httpHeaders, HttpStatus.OK); } + + @Override + @PreAuthorize("hasAuthority('" + READ_RULES + "')") + public ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + List summaries = componentMappingService.getMetaDataByDossierTemplateId( + dossierTemplateId); + List componentMappingMetadataModelList = componentMappingMapper.toModelList(summaries); + return new ComponentMappingSummary(dossierTemplateId, componentMappingMetadataModelList); + } + + + @Override + @SneakyThrows + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, char delimiter) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + String nameToUse = Strings.isNullOrEmpty(name) ? file.getName().split("\\.")[0] : name; + + if (Strings.isNullOrEmpty(nameToUse)) { + throw new BadRequestException("The provided file name is not valid!"); + } + + Path mappingFile = saveToFile(file); + String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); + + com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, + nameToUse, + fileName, + delimiter, + encoding, + mappingFile.toFile()); + + Files.deleteIfExists(mappingFile); + + return componentMappingMapper.toModel(metaData); + } + + + @Override + @SneakyThrows + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String encoding, char delimiter) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + Path mappingFile = saveToFile(file); + + com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId, + componentMappingId, + encoding, + delimiter, + mappingFile.toFile()); + + Files.deleteIfExists(mappingFile); + + return componentMappingMapper.toModel(resultMetaData); + } + + + @Override + @PreAuthorize("hasAuthority('" + READ_RULES + "')") + public ResponseEntity downloadMapping(String dossierTemplateId, String componentMappingId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + ComponentMappingDownloadModel mappingDownloadModel = componentMappingService.getMappingForDownload(dossierTemplateId, componentMappingId); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.TEXT_PLAIN); + + httpHeaders.add("Content-Disposition", + "attachment" + + "; filename*=" + + mappingDownloadModel.encoding().toLowerCase(Locale.US) + + "''" + + StringEncodingUtils.urlEncode(mappingDownloadModel.fileName())); + + return new ResponseEntity<>(mappingDownloadModel.mappingFileResource(), httpHeaders, HttpStatus.OK); + } + + + @Override + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ResponseEntity deleteMapping(String dossierTemplateId, String componentMappingId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + componentMappingService.delete(dossierTemplateId, componentMappingId); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + + @SneakyThrows + private static Path saveToFile(MultipartFile file) { + + Path mappingFile = Files.createTempFile(file.getName(), ".csv"); + + try (var out = new FileOutputStream(mappingFile.toFile())) { + out.write(file.getBytes()); + } + return mappingFile; + } + } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java new file mode 100644 index 000000000..b22db8eb7 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java @@ -0,0 +1,184 @@ +package com.iqser.red.persistence.service.v2.external.api.impl.controller; + +import java.time.OffsetDateTime; +import java.util.Map; +import java.util.stream.Collectors; + +import org.quartz.JobDataMap; +import org.quartz.JobKey; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.HttpStatus; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.WebDataBinder; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.multipart.support.MissingServletRequestPartException; + +import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import com.iqser.red.commons.spring.ErrorMessage; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.knecon.fforesight.tenantcommons.TenantContext; +import com.mchange.rmi.NotAuthorizedException; + +import io.swagger.v3.oas.annotations.Hidden; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RestControllerAdvice +@RequiredArgsConstructor +@Order(Ordered.HIGHEST_PRECEDENCE) +public class ExternalControllerAdviceV2 { + + private final Scheduler scheduler; + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.NOT_FOUND) + @ExceptionHandler(value = NotFoundException.class) + public ErrorMessage handleContentNotFoundException(NotFoundException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + /* error handling */ + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler(value = BadRequestException.class) + public ErrorMessage handleBadRequestException(BadRequestException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.CONFLICT) + @ExceptionHandler(value = {ConflictException.class}) + protected ErrorMessage handleConflictException(ConflictException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.FORBIDDEN) + @ExceptionHandler({AccessDeniedException.class}) + public ErrorMessage handleAccessDeniedException(AccessDeniedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.UNAUTHORIZED) + @ExceptionHandler({NotAuthorizedException.class}) + public ErrorMessage handleNotAuthorizedException(NotAuthorizedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.FORBIDDEN) + @ExceptionHandler({NotAllowedException.class}) + public ErrorMessage handleNotAllowedException(NotAllowedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + @ExceptionHandler({org.springframework.security.acls.model.NotFoundException.class}) + public ErrorMessage handleACLNotFound(org.springframework.security.acls.model.NotFoundException e) { + + // in case this error occurs on a rest request / force trigger the sync job + try { + scheduler.triggerJob(new JobKey("SyncUserPermissionsJob"), new JobDataMap(Map.of("tenantId", TenantContext.getTenantId()))); + } catch (SchedulerException ex) { + log.debug("Failed to force trigger SyncUserPermissionsJob", ex); + } + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler({MethodArgumentNotValidException.class}) + public ErrorMessage handleMethodArgumentNotValidException(MethodArgumentNotValidException e) { + + var errorList = e.getFieldErrors(); + String errorListAsString = errorList.stream() + .map(fieldError -> fieldError.getField() + ": " + fieldError.getDefaultMessage()) + .collect(Collectors.joining(", ")); + return new ErrorMessage(OffsetDateTime.now(), String.format("You have empty/wrong formatted parameters: %s", errorListAsString)); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler({MissingServletRequestPartException.class}) + public ErrorMessage handleMissingServletRequestPartException(MissingServletRequestPartException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler({HttpMessageNotReadableException.class}) + public ErrorMessage handleHttpMessageNotReadableException(HttpMessageNotReadableException e) { + + var cause = e.getCause(); + if (cause instanceof InvalidFormatException) { + InvalidFormatException invalidFormatException = (InvalidFormatException) cause; + + Class targetType = invalidFormatException.getTargetType(); + if (targetType != null && targetType.isEnum()) { + return new ErrorMessage(OffsetDateTime.now(), String.format("Unsupported value for %s", targetType.getSimpleName())); + } + + return new ErrorMessage(OffsetDateTime.now(), cause.getMessage()); + } + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Order(10000) + public static class BinderControllerAdvice { + + @InitBinder + public void setAllowedFields(WebDataBinder dataBinder) { + // This code protects Spring Core from a "Remote Code Execution" attack (dubbed "Spring4Shell"). + // By applying this mitigation, you prevent the "Class Loader Manipulation" attack vector from firing. + // For more details, see this post: https://www.lunasec.io/docs/blog/spring-rce-vulnerabilities/ + String[] denylist = new String[]{"class.*", "Class.*", "*.class.*", "*.Class.*"}; + dataBinder.setDisallowedFields(denylist); + } + + } + +} diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMappingMapper.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMappingMapper.java new file mode 100644 index 000000000..975db2bf8 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMappingMapper.java @@ -0,0 +1,28 @@ +package com.iqser.red.persistence.service.v2.external.api.impl.mapper; + +import java.util.List; + +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; + +@Mapper +public interface ComponentMappingMapper { + + ComponentMappingMapper INSTANCE = Mappers.getMapper(ComponentMappingMapper.class); + + + ComponentMappingMetadataModel toModel(ComponentMappingMetadata componentMappingMetadata); + + + List toModelList(List componentMappingMetadata); + + + ComponentMappingMetadata toDto(ComponentMappingMetadataModel componentMappingMetadataModel); + + + List toDtoList(List componentMappingMetadataModels); + +} diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingMetadataModel.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingMetadataModel.java new file mode 100644 index 000000000..b8da5546b --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingMetadataModel.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.persistence.service.v2.api.external.model; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ComponentMappingMetadataModel { + + String id; + String name; + String fileName; + Integer version; + List columnLabels; + Integer numberOfLines; + String encoding; + char delimiter; + +} diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingSummary.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingSummary.java new file mode 100644 index 000000000..d80c46346 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingSummary.java @@ -0,0 +1,22 @@ +package com.iqser.red.service.persistence.service.v2.api.external.model; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ComponentMappingSummary { + + String dossierTemplateId; + List componentMappingList; + +} diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java index ca0cdbec1..a67b78e9c 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java @@ -8,20 +8,29 @@ import static com.iqser.red.service.persistence.service.v2.api.external.resource import static com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource.DOSSIER_TEMPLATE_PATH; import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_ID_PARAM; import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_ID_PATH_VARIABLE; -import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_PATH; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.multipart.MultipartFile; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents; import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponentsList; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -29,7 +38,8 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; @ResponseStatus(value = HttpStatus.OK) public interface ComponentResource { - String PATH = ExternalApiConstants.BASE_PATH + DOSSIER_TEMPLATE_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + DOSSIER_PATH + DOSSIER_ID_PATH_PARAM + FILE_PATH; + String PATH = ExternalApiConstants.BASE_PATH + DOSSIER_TEMPLATE_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE; + String FILE_PATH = PATH + DOSSIER_PATH + DOSSIER_ID_PATH_PARAM + FileResource.FILE_PATH; String COMPONENTS_PATH = "/components"; @@ -39,13 +49,13 @@ public interface ComponentResource { String INCLUDE_DETAILS_DESCRIPTION = """ A toggle to decide whether to include detailed component information in the response: - - true: The component object's field componentDetails stores detailed information about the source of its respective value(s). - false (default): The component object does not contain a field componentDetails. """; - @GetMapping(value = PATH + FILE_ID_PATH_VARIABLE + COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) + + @GetMapping(value = FILE_PATH + FILE_ID_PATH_VARIABLE + COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) @Operation(summary = "Returns the components for a file", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) FileComponents getComponents(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @@ -54,11 +64,13 @@ public interface ComponentResource { @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails); - @GetMapping(value = PATH + BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) + @GetMapping(value = FILE_PATH + BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) @Operation(summary = "Returns the components for all files of a dossier", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) FileComponentsList getComponentsOfDossier(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier that contains the file.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId, @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails); + + } diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java index 78c1ec44e..7257cdd41 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java @@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.service.v2.api.external.resource; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList; import io.swagger.v3.oas.annotations.Operation; @@ -13,9 +15,11 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.ResponseBody; @@ -33,15 +37,20 @@ public interface DossierTemplateResource { String ENTITY_RULES_PATH = "/entity-rules"; String COMPONENT_RULES_PATH = "/component-rules"; + String COMPONENT_MAPPINGS_PATH = "/component-mappings"; String FILE_ATTRIBUTE_DEFINITIONS_PATH = "/file-attribute-definitions"; String DOSSIER_TEMPLATE_ID_PARAM = "dossierTemplateId"; String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID_PARAM + "}"; - String RULE_FILE_TYPE_PARAMETER_NAME = "ruleFileType"; + + String COMPONENT_MAPPING_ID_PARAM = "componentMappingId"; + String COMPONENT_MAPPING_ID_PATH_VARIABLE = "/{" + COMPONENT_MAPPING_ID_PARAM + "}"; String DRY_RUN_PARAM = "dryRun"; - + String ENCODING_PARAM = "encoding"; + String DELIMITER_PARAM = "delimiter"; + String MAPPING_NAME_PARAM = "name"; @GetMapping(value = PATH, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Lists all existing DossierTemplates.", description = "None") @@ -60,8 +69,8 @@ public interface DossierTemplateResource { @Operation(summary = "Upload a component or entity rules file in drools format for a specific DossierTemplate.") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Rules upload successful."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found."), @ApiResponse(responseCode = "400", description = "Uploaded rules could not be verified."), @ApiResponse(responseCode = "422", description = "Uploaded rules could not be compiled.")}) ResponseEntity uploadEntityRules(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, - @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, - @Parameter(name = DRY_RUN_PARAM, description = "If true rules will be only validated not stored.") @RequestParam(value = DRY_RUN_PARAM, required = false, defaultValue = "false") boolean dryRun); + @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @Parameter(name = DRY_RUN_PARAM, description = "If true rules will be only validated not stored.") @RequestParam(value = DRY_RUN_PARAM, required = false, defaultValue = "false") boolean dryRun); @ResponseBody @@ -96,4 +105,51 @@ public interface DossierTemplateResource { @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "File attribute definitions returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) FileAttributeDefinitionList getFileAttributeDefinitions(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId); + + @Operation(summary = "Get the component mapping summaries of a DossierTemplate.", description = "None") + @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping views returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) + ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId); + + + @Operation(summary = "Upload a new component mapping to a DossierTemplate.", description = "None") + @PostMapping(value = PATH + + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + + COMPONENT_MAPPINGS_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping uploaded successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + ComponentMappingMetadataModel uploadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, + @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name, + @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, + @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); + + + @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None") + @PutMapping(value = PATH + + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + + COMPONENT_MAPPINGS_PATH + + COMPONENT_MAPPING_ID_PATH_VARIABLE, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping updated successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + ComponentMappingMetadataModel updateMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, + @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId, + @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, + @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); + + + @ResponseBody + @ResponseStatus(value = HttpStatus.OK) + @Operation(summary = "Returns file containing the specified mapping as a file.") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE) + ResponseEntity downloadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId); + + + @ResponseBody + @ResponseStatus(value = HttpStatus.OK) + @Operation(summary = "Deletes a specified mapping.") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + @DeleteMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE) + ResponseEntity deleteMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId); + } diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml index 4ccd4fb90..9691da1d5 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml @@ -276,7 +276,7 @@ paths: tags: - 1. Dossier Templates description: | - Retrieves a collection of file attribute definitions associated with a specific dossier template. Each file + Use this endpoint to retrieves a collection of file attribute definitions associated with a specific dossier template. Each file attribute definition includes details such as attribute type, name, and other relevant metadata. This endpoint is useful for clients needing to understand what attributes are expected or allowed for files under a particular dossier template. @@ -290,6 +290,214 @@ paths: $ref: '#/components/schemas/FileAttributeDefinitionList' description: | Successfully returned the file attribute definitions for the specified dossier template. + /api/dossier-templates/{dossierTemplateId}/component-mappings: + get: + operationId: listAllMappings + tags: + - 1. Dossier Templates + summary: Returns a list of all existing component mappings in a dossier template + description: | + Use this endpoint to retrieve a summary of all component mappings associated with a specific DossierTemplate. + + The summary consists of the stored metadata of a component mapping file. + This endpoint is useful for clients to understand the available mappings under a particular DossierTemplate. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMappingList' + description: | + Successfully returned the component mapping summary for the specified dossier template. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + post: + operationId: uploadMapping + summary: Upload a new component mapping to a DossierTemplate. + description: | + Use this endpoint to upload a new component mapping to a specific DossierTemplate. + + #### File Requirements + - **Format:** The file must be in CSV (comma-separated values) format. + - **Header Row:** The first row should contain the column labels. + - **Data Consistency:** All rows must have the same number of columns to ensure rectangular data structure. + + #### Sorting and Performance + - **Sorting:** Rows are automatically sorted by the values in each column, from left to right, to enhance lookup speed. + - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance. + + #### Customization Options + - Users can specify the delimiter and encoding used in the CSV file. + + #### Usage + - The component mapping file can be utilized in component rules to relate components to existing master data. + + #### Example + + | search_value | mapped_value | + |--------------|--------------| + | Alice | Manager | + | Bob | Developer | + | Charlie | Analyst | + tags: + - 1. Dossier Templates + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/mappingName' + - $ref: '#/components/parameters/encoding' + - $ref: '#/components/parameters/delimiter' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMapping' + description: | + Component mapping uploaded successfully and returned the component mapping metadata. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + + /api/dossier-templates/{dossierTemplateId}/component-mappings/{comonentMappingId}: + get: + operationId: downloadMappingFile + tags: + - 1. Dossier Templates + summary: Download a specific component mapping file of a specific dossier template. + description: | + Use this endpoint to download a specific component mapping file of a designated DossierTemplate. + + - The file retains its original name and encoding as when it was uploaded. + - The sorting of the file may have changed to enable faster lookups. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + responses: + "200": + headers: + Content-Disposition: + schema: + type: string + example: attachment; filename*=mapping.csv + content: + text/plain: + schema: + type: string + description: | + Successfully downloaded the requested component mapping. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + put: + operationId: updateMapping + summary: Update an existing component mapping of a DossierTemplate. + description: | + Use this endpoint to update an existing component mapping of a specific dossier template. + + #### File Requirements + - **Format:** The file must be in CSV (comma-separated values) format. + - **Header Row:** The first row should contain the column labels. + - **Data Consistency:** All rows must have the same number of columns to ensure rectangular data structure. + + #### Sorting and Performance + - **Sorting:** Rows are automatically sorted by the values in each column, from left to right, to enhance lookup speed. + - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance. + + #### Customization Options + - Users can specify the delimiter and encoding used in the CSV file. + tags: + - 1. Dossier Templates + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + - $ref: '#/components/parameters/encoding' + - $ref: '#/components/parameters/delimiter' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMapping' + description: | + Component mapping uploaded successfully and returned the component mapping metadata. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + delete: + operationId: deleteMappingFile + tags: + - 1. Dossier Templates + summary: Delete a specific component mapping file of a specific dossier template. + description: | + Use this endpoint to delete a specific component mapping file of a designated dossier template. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + responses: + "204": + description: | + Successfully deleted the requested component mapping. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' /api/dossier-templates/{dossierTemplateId}/dossiers: get: operationId: getDossiers @@ -304,7 +512,7 @@ paths: Use this endpoint to fetch the required dossiers before performing actions on specific ones. Use the query parameters to modify the response. E.g., set the `includeArchivedDossiers` parameter to `true` so that the response also contains - *archived* dossiers. + *archived* dossiers. parameters: - $ref: '#/components/parameters/dossierTemplateId' - $ref: '#/components/parameters/includeActiveDossiers' @@ -792,14 +1000,14 @@ components: responses: "400": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: | The request was malformed or invalid. Double-check the request structure or parameters. "401": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: | @@ -807,21 +1015,28 @@ components: resource. This error can happen if the access token was revoked or has expired. "403": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: | Forbidden. Your credentials are valid, but you do not have permission to access this resource. "404-dossier-template": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: | Dossier template not found. This happens if the requested dossier template does not exist. + "404-mapping": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Dossier template or component mapping not found. This happens if the requested dossier template or component mapping does not exist. "404-dossier": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: | @@ -831,7 +1046,7 @@ components: for a previously existing dossier only if it is actually deleted permanently. "404-file": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: | @@ -841,28 +1056,28 @@ components: only if the file is deleted permanently. "409-dossier-conflict": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: | Name conflict: The provided name is already in use by another dossier. It needs to be unique in the scope of your workspace. 422-rules: content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/RuleValidation' description: | Invalid rules file: There were validation errors, the rules file is unprocessable. "429": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: | Too many requests have been made in a given time frame. Rate limiting is in place to prevent abuse. "500": content: - '*/*': + 'application/json': schema: $ref: '#/components/schemas/ErrorMessage' description: Internal Server Error. An unexpected error occurred on the server side. Please try again later or contact support. @@ -876,6 +1091,15 @@ components: style: simple explode: false description: The identifier of a dossier template + componentMappingId: + name: componentMappingId + in: path + required: true + schema: + type: string + style: simple + explode: false + description: The identifier of a component mapping dryRun: name: dryRun in: query @@ -888,6 +1112,41 @@ components: description: | A toggle to activate the dry-run mode: If set to `false` (default), the request will update the system. If set to `true`, the request will just be evaluated without actual changes in the system. + encoding: + name: encoding + required: false + in: query + schema: + type: string + enum: + - UTF-8 + - UTF-16 + - UTF_16BE + - UTF_16LE + - ISO-8859-1 + - US-ASCII + example: UTF-8 + default: UTF-8 + description: "An identifier for the used encoding of the file. Only java's standard charsets are supported." + delimiter: + name: delimiter + required: false + in: query + schema: + type: string + minLength: 1 + maxLength: 1 + example: ',' + default: ',' + description: "The delimiter used as a separator in a csv file." + mappingName: + name: name + required: false + in: query + schema: + type: string + example: "MasterDataMapping" + description: "The name with which the mapping should be associated with. If none is provided, the file name will be used." dossierId: name: dossierId in: path @@ -1848,6 +2107,108 @@ components: - daadea5f-917b-482a-b7d2-e65afe8f80ca - 8130acf6-4910-4123-827c-caacd8111402 dossierStatusId: b8280985-f558-43c0-852a-a3fa86803548 + ComponentMapping: + description: | + The `ComponentMapping` object represents the metadata of a component mapping csv file. These CSV files may be used in the component rules to relate components to an existing knowledge base. + type: object + properties: + id: + description: | + A unique identifier for the component mapping metadata. It's generated automatically. Use this to retrieve a specific component mapping file. + type: string + name: + description: | + The name of the component mapping. + type: string + fileName: + description: | + The name of the uploaded component mapping file. + type: string + version: + description: | + The version of the file. It is incremented with each update to the file. + type: integer + columnLabels: + description: | + A list of column labels found in the component mapping file. + type: array + items: + type: string + numberOfLines: + description: | + The number of lines in the component mapping file. + type: integer + format: int32 + encoding: + description: | + The encoding used for the component mapping file. + type: string + delimiter: + description: | + The delimiter used for separating values in the component mapping file. + type: string + required: + - id + - name + - fileName + - columnLabels + - numberOfLines + - encoding + - delimiter + example: + id: 24ff9c3c-4863-4aea-8eda-cab8838b9192 + name: MasterDataMapping + fileName: master_data.csv + columnLabels: + - Column1 + - Column2 + - Column3 + numberOfLines: 100 + encoding: UTF-8 + delimiter: ',' + ComponentMappingList: + description: | + The `ComponentMappingList` object represents a collection of ComponentMapping. + type: object + example: + dossierTemplateId: 1e07cde0-d36a-4ab7-b389-494ca694a0cb + componentMappingList: + - id: 24ff9c3c-4863-4aea-8eda-cab8838b9192 + name: MasterDataMapping + fileName: master_data.csv + columnLabels: + - Column1 + - Column2 + - Column3 + numberOfLines: 100 + encoding: UTF-8 + delimiter: ',' + - id: 2e07cde0-d36a-4ab7-b389-494ca694a0cb + name: RegulationNameMapping + fileName: regulation-names.csv + columnLabels: + - Year + - Title + - Identifier + - Name + numberOfLines: 150 + encoding: UTF-16 + delimiter: ; + properties: + dossierTemplateId: + description: | + The identifier of the dossier template associated with the ComponentMappingList. + type: string + format: uuid + componentMappingList: + description: | + A list of component mapping metadata associated with this dossier template. + type: array + items: + $ref: "#/components/schemas/ComponentMapping" + required: + - dossierTemplateId + - componentMappingList DossierTemplate: description: | The `DossierTemplate` object represents the blueprint for creating and diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java index 7d530b5d5..846312fc8 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java @@ -53,6 +53,7 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP public void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AnalyzeResult analyzeResult) { + log.info("Received analysis result {}", analyzeResult); fileStatusProcessingUpdateService.analysisSuccessful(dossierId, fileId, analyzeResult); } diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index dbf220411..6ec669f08 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -61,6 +61,7 @@ dependencies { api("com.opencsv:opencsv:5.4") api("org.springframework.cloud:spring-cloud-starter-openfeign:${springCloudVersion}") api("commons-validator:commons-validator:1.7") + api("com.opencsv:opencsv:5.9") implementation("org.mapstruct:mapstruct:1.5.5.Final") annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java new file mode 100644 index 000000000..67f27f504 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java @@ -0,0 +1,78 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; + +import org.hibernate.annotations.Fetch; +import org.springframework.data.annotation.LastModifiedDate; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; + +import jakarta.persistence.CollectionTable; +import jakarta.persistence.Column; +import jakarta.persistence.ElementCollection; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.Id; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +@Data +@Entity +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Table(name = "component_mappings") +public class ComponentMappingEntity { + + @Id + @Column(name = "id") + String id; + + @NonNull + @ManyToOne(fetch = FetchType.LAZY) + private DossierTemplateEntity dossierTemplate; + + @NonNull + @Builder.Default + String storageId = ""; + + @NonNull + @Builder.Default + String name = ""; + + @NonNull + @Builder.Default + String fileName = ""; + + @NonNull + @Builder.Default + Integer version = -1; + + @LastModifiedDate + OffsetDateTime changedDate; + + @NonNull + @Builder.Default + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name = "component_mapping_column_labels") + List columnLabels = new ArrayList<>(); + + @NonNull + @Builder.Default + Integer numberOfLines = 0; + + @NonNull + @Builder.Default + String encoding = "UTF-8"; + + @Builder.Default + char delimiter = ','; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java index fc3d0f1a3..030752a2b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java @@ -69,8 +69,8 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation { @Column private OffsetDateTime softDeletedTime; - @ElementCollection(fetch = FetchType.EAGER) @Fetch(FetchMode.SUBSELECT) + @ElementCollection(fetch = FetchType.EAGER) private List positions = new ArrayList<>(); @ManyToOne(fetch = FetchType.LAZY) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 61fda011a..4ba0806b4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -14,8 +14,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; import jakarta.persistence.CascadeType; +import jakarta.persistence.CollectionTable; import jakarta.persistence.Column; import jakarta.persistence.Convert; +import jakarta.persistence.ElementCollection; import jakarta.persistence.Entity; import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; @@ -197,6 +199,9 @@ public class FileEntity { @Column private OffsetDateTime errorTimestamp; + @ElementCollection + private List componentMappingVersions; + public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java new file mode 100644 index 000000000..f50700668 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.dossier; + +import jakarta.persistence.Embeddable; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Embeddable +@AllArgsConstructor +@NoArgsConstructor +public class FileEntityComponentMappingVersionEntity { + + private String name; + private Integer version; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java new file mode 100644 index 000000000..7fa8ff3be --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.persistence.management.v1.processor.mapper; + +import java.util.List; + +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; + +@Mapper +public interface ComponentMappingEntityMapper { + + ComponentMappingEntityMapper INSTANCE = Mappers.getMapper(ComponentMappingEntityMapper.class); + + + ComponentMappingMetadata toComponentMappingMetaData(ComponentMappingEntity componentMappingMetaData); + + + List toComponentMappingMetaDataList(List componentMappingEntities); + + + ComponentMappingEntity toComponentMappingEntity(ComponentMappingMetadata componentMappingSummary); + + + List toComponentMappingEntityList(List componentMappingSummary); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMapping.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMapping.java new file mode 100644 index 000000000..c1528c29d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMapping.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.persistence.management.v1.processor.model; + +import java.io.File; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; + +public record ComponentMapping(ComponentMappingMetadata metaData, File file) { + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMappingDownloadModel.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMappingDownloadModel.java new file mode 100644 index 000000000..399c9d76c --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMappingDownloadModel.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.persistence.management.v1.processor.model; + +import org.springframework.core.io.InputStreamResource; + +public record ComponentMappingDownloadModel(InputStreamResource mappingFileResource, String encoding, String fileName) { + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/DownloadJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/DownloadJob.java index b42537b58..91fc5b18c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/DownloadJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/DownloadJob.java @@ -14,6 +14,5 @@ public class DownloadJob { private String userId; private String storageId; - private Boolean includeUnprocessed; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java new file mode 100644 index 000000000..43a75883b --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java @@ -0,0 +1,104 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.util.List; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; +import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ComponentMappingRepository; +import com.iqser.red.storage.commons.service.StorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; + +@Service +@Transactional +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class ComponentMappingPersistenceService { + + StorageService storageService; + + ComponentMappingRepository repository; + + + public List getByDossierTemplateId(String dossierTemplateId) { + + return repository.findByDossierTemplateId(dossierTemplateId); + } + + + public void deleteById(String dossierTemplateId, String id) { + + ComponentMappingEntity entity = getEntityById(dossierTemplateId, id); + delete(entity); + } + + + public ComponentMappingEntity getEntityById(String dossierTemplateId, String id) { + + return repository.findByIdAndDossierTemplateId(id, dossierTemplateId) + .orElseThrow(() -> new NotFoundException("ComponentMapping with id " + id + " not found!")); + } + + + private void delete(ComponentMappingEntity entity) { + + repository.delete(entity); + storageService.deleteObject(TenantContext.getTenantId(), entity.getStorageId()); + } + + + public void deleteByDossierTemplateId(String dossierTemplateId) { + + repository.findByDossierTemplateId(dossierTemplateId) + .forEach(this::delete); + } + + + @SneakyThrows + public void updateOrCreate(String storageId, File mappingFile, ComponentMappingEntity entity) { + + entity.setChangedDate(OffsetDateTime.now()); + repository.saveAndFlush(entity); + try (var in = new FileInputStream(mappingFile)) { + storageService.storeObject(TenantContext.getTenantId(), storageId, in); + } + } + + + public boolean existsByNameAndDossierTemplateId(String name, String dossierTemplateId) { + + return repository.existsByNameAndDossierTemplateId(name, dossierTemplateId); + } + + + @SneakyThrows + public ComponentMappingDownloadModel getMappingFileForDownload(String dossierTemplateId, String componentMappingId) { + + var entity = getEntityById(dossierTemplateId, componentMappingId); + if (!storageService.objectExists(TenantContext.getTenantId(), entity.getStorageId())) { + throw new NotFoundException("ComponentMapping with id " + componentMappingId + " does not exist!"); + } + return new ComponentMappingDownloadModel(storageService.getObject(TenantContext.getTenantId(), entity.getStorageId()), entity.getEncoding(), entity.getFileName()); + } + + + public File downloadMappingFileToFolder(String storageId, String fileName, Path outputDir) { + + File outputFile = outputDir.resolve(fileName).toFile(); + storageService.downloadTo(TenantContext.getTenantId(), storageId, outputFile); + return outputFile; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java new file mode 100644 index 000000000..cfd7c3483 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java @@ -0,0 +1,225 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.nio.charset.Charset; +import java.nio.charset.IllegalCharsetNameException; +import java.nio.charset.UnsupportedCharsetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.mapper.ComponentMappingEntityMapper; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.CSVWriter; +import com.opencsv.exceptions.CsvException; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; + +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class ComponentMappingService { + + ComponentMappingPersistenceService componentMappingPersistenceService; + DossierTemplatePersistenceService dossierTemplatePersistenceService; + ComponentMappingEntityMapper mappingEntityMapper = ComponentMappingEntityMapper.INSTANCE; + + static CSVSorter CSV_SORTER = new CSVSorter(); + + + public List getMetaDataByDossierTemplateId(String dossierTemplateId) { + + List entities = componentMappingPersistenceService.getByDossierTemplateId(dossierTemplateId); + return mappingEntityMapper.toComponentMappingMetaDataList(entities); + } + + + public ComponentMappingMetadata getMetaData(String dossierTemplateId, String mappingId) { + + return mappingEntityMapper.toComponentMappingMetaData(componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId)); + } + + + @SneakyThrows + public ComponentMappingMetadata update(String dossierTemplateId, String mappingId, String encoding, char delimiter, File mappingFile) { + + ComponentMappingEntity entity = componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId); + + return updateOrCreate(entity, encoding, delimiter, mappingFile); + } + + + @SneakyThrows + public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile) { + + if (componentMappingPersistenceService.existsByNameAndDossierTemplateId(name, dossierTemplateId)) { + throw new BadRequestException("A mapping with this name already exists in the dossier template!"); + } + + String id = UUID.randomUUID().toString(); + String storageId = buildStorageId(dossierTemplateId, id, name, fileName); + ComponentMappingEntity entity = ComponentMappingEntity.builder() + .id(id) + .dossierTemplate(dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId)) + .storageId(storageId) + .name(name) + .fileName(fileName) + .build(); + + return updateOrCreate(entity, encoding, delimiter, mappingFile); + } + + + @SneakyThrows + private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity, String encoding, char delimiter, File mappingFile) { + + Charset charset = resolveCharset(encoding); + + CsvStats stats = sortCSVFile(delimiter, mappingFile, charset); + + entity.setDelimiter(delimiter); + entity.setEncoding(encoding); + entity.setNumberOfLines(stats.numberOfLines()); + entity.setColumnLabels(stats.columnLabels()); + entity.setVersion(entity.getVersion() + 1); + + componentMappingPersistenceService.updateOrCreate(entity.getStorageId(), mappingFile, entity); + + return mappingEntityMapper.toComponentMappingMetaData(entity); + } + + + private static Charset resolveCharset(String encoding) { + + try { + return Charset.forName(encoding); + } catch (IllegalCharsetNameException e) { + throw new BadRequestException("Invalid character encoding: " + encoding); + } catch (UnsupportedCharsetException e) { + throw new BadRequestException("Unsupported character encoding: " + encoding); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Encoding can't be null."); + } + } + + + private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws CsvException, BadRequestException, IOException { + + Path tempFile = Files.createTempFile("mapping", ".tmp"); + + Files.move(mappingFile.toPath(), tempFile, StandardCopyOption.REPLACE_EXISTING); + + String[] columnLabels; + int numberOfLines = 0; + try (Reader fileReader = new FileReader(tempFile.toFile(), charset);// + CSVReader reader = buildReader(fileReader, delimiter);// + CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset))) { + + List rows = reader.readAll(); + + columnLabels = rows.remove(0); + + numberOfLines = (int) reader.getLinesRead() - 1; + + rows.sort(CSV_SORTER); + + writer.writeNext(columnLabels); + + writer.writeAll(rows); + + } catch (IOException e) { + throw new BadRequestException("Error while sorting the csv file", e); + } + + Files.deleteIfExists(tempFile); + + return new CsvStats(Arrays.asList(columnLabels), numberOfLines); + } + + + private static CSVReader buildReader(Reader reader, char delimiter) throws IOException { + + return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build(); + } + + + public void delete(String dossierTemplateId, String componentMappingId) { + + componentMappingPersistenceService.deleteById(dossierTemplateId, componentMappingId); + } + + + public ComponentMappingDownloadModel getMappingForDownload(String dossierTemplateId, String componentMappingId) { + + return componentMappingPersistenceService.getMappingFileForDownload(dossierTemplateId, componentMappingId); + } + + + @SneakyThrows + public List getMappingFilesByDossierTemplateId(String dossierTemplateId, Path outputDir) { + + var entities = componentMappingPersistenceService.getByDossierTemplateId(dossierTemplateId); + + return entities.stream() + .map(mappingEntityMapper::toComponentMappingMetaData) + .map(metaData -> downloadFileAndCreateMapping(outputDir, metaData)) + .toList(); + + } + + + private ComponentMapping downloadFileAndCreateMapping(Path outputDir, ComponentMappingMetadata metaData) { + + File mappingFile = componentMappingPersistenceService.downloadMappingFileToFolder(metaData.getStorageId(), metaData.getFileName(), outputDir); + return new ComponentMapping(metaData, mappingFile); + } + + + public static String buildStorageId(String dossierTemplateId, String id, String name, String fileName) { + + return dossierTemplateId + "/" + id + "_" + name + "_" + fileName; + } + + + private static class CSVSorter implements Comparator { + + @Override + public int compare(String[] line1, String[] line2) { + + for (int column = 0; column < line1.length; column++) { + if (!line1[column].equals(line2[column])) { + return line1[column].compareTo(line2[column]); + } + } + + return 0; + } + + } + + private record CsvStats(List columnLabels, int numberOfLines) { + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java index e9bc1c8cc..f09b35fb7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -10,6 +12,7 @@ import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; +import org.springframework.util.FileSystemUtils; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity; @@ -20,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService; @@ -40,29 +44,33 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; import com.knecon.fforesight.tenantcommons.TenantContext; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DossierTemplateCloneService { - private final DossierTemplateRepository dossierTemplateRepository; - private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; - private final RulesPersistenceService rulesPersistenceService; - private final DossierTemplatePersistenceService dossierTemplatePersistenceService; - private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService; - private final DictionaryPersistenceService dictionaryPersistenceService; - private final EntryPersistenceService entryPersistenceService; - private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; - private final ReportTemplatePersistenceService reportTemplatePersistenceService; - private final ColorsService colorsService; - private final StorageService storageService; - private final DossierStatusPersistenceService dossierStatusPersistenceService; - private final WatermarkService watermarkService; - private final FileManagementStorageService fileManagementStorageService; + DossierTemplateRepository dossierTemplateRepository; + LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + RulesPersistenceService rulesPersistenceService; + DossierTemplatePersistenceService dossierTemplatePersistenceService; + DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService; + DictionaryPersistenceService dictionaryPersistenceService; + EntryPersistenceService entryPersistenceService; + FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; + ReportTemplatePersistenceService reportTemplatePersistenceService; + ColorsService colorsService; + StorageService storageService; + DossierStatusPersistenceService dossierStatusPersistenceService; + WatermarkService watermarkService; + FileManagementStorageService fileManagementStorageService; + ComponentMappingService componentMappingService; public DossierTemplateEntity cloneDossierTemplate(String dossierTemplateId, CloneDossierTemplateRequest cloneDossierTemplateRequest) { @@ -129,6 +137,8 @@ public class DossierTemplateCloneService { // set the watermark configurations cloneWatermarks(dossierTemplateId, clonedDossierTemplate.getId()); + cloneComponentMappings(dossierTemplateId, clonedDossierTemplate.getId()); + return clonedDossierTemplate; } @@ -153,6 +163,23 @@ public class DossierTemplateCloneService { } + @SneakyThrows + private void cloneComponentMappings(String dossierTemplateId, String clonedDossierTemplateId) { + + Path dir = Files.createTempDirectory("componentMappingsForClone"); + List componentMappings = componentMappingService.getMappingFilesByDossierTemplateId(dossierTemplateId, dir); + for (ComponentMapping componentMapping : componentMappings) { + componentMappingService.create(clonedDossierTemplateId, + componentMapping.metaData().getName(), + componentMapping.metaData().getFileName(), + componentMapping.metaData().getDelimiter(), + componentMapping.metaData().getEncoding(), + componentMapping.file()); + } + FileSystemUtils.deleteRecursively(dir); + } + + private void cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) { var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java index fb0c6d290..012946acd 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java @@ -8,14 +8,17 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -60,6 +63,7 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa import com.iqser.red.service.persistence.management.v1.processor.utils.FileUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus; @@ -70,6 +74,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ComponentMappingImportModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ExportFilename; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportTemplateResult; @@ -81,6 +86,7 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -109,6 +115,7 @@ public class DossierTemplateImportService { private final StorageService storageService; private final ObjectMapper objectMapper = new ObjectMapper(); private final FileManagementServiceSettings settings; + private final ComponentMappingService componentMappingService; public String importDossierTemplate(ImportDossierTemplateRequest request) { @@ -145,7 +152,8 @@ public class DossierTemplateImportService { Map> typeEntriesMap = new HashMap<>(); Map> typeFalsePositivesMap = new HashMap<>(); Map> typeFalseRecommendationsMap = new HashMap<>(); - + Map mappingDataMap = new HashMap<>(); + List mappingMetadataList = new LinkedList<>(); while ((ze = zis.getNextZipEntry()) != null) { log.debug("---> " + ze.getName() + " ---- " + ze.isDirectory()); totalEntryArchive++; @@ -274,12 +282,22 @@ public class DossierTemplateImportService { reportTemplateFilenameList = reportTemplateList.stream() .map(rt -> rt.isMultiFileReport() ? rt.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : rt.getFileName()) .toList(); + } else if (ze.getName().startsWith(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename())) { + if (ze.getName().contains(ExportFilename.COMPONENT_MAPPINGS_FILE.getFilename())) { + mappingMetadataList = objectMapper.readValue(bytes, new TypeReference<>() { + }); + } else if (ze.getName().endsWith(".csv")) { + String fileName = ze.getName().replace(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename() + "/", ""); + mappingDataMap.put(fileName, bytes); + } + } else { reportTemplateBytesMap.put(ze.getName(), bos); } bos.close(); } } + importTemplateResult.setEntries(typeEntriesMap); importTemplateResult.setFalsePositives(typeFalsePositivesMap); importTemplateResult.setFalseRecommendations(typeFalseRecommendationsMap); @@ -298,6 +316,13 @@ public class DossierTemplateImportService { } } + + for (var metadata : mappingMetadataList) { + String fileName = metadata.getName() + ".csv"; + if (mappingDataMap.containsKey(fileName)) { + importTemplateResult.componentMappings.add(new ComponentMappingImportModel(metadata, mappingDataMap.get(fileName))); + } + } if (importTemplateResult.getDossierTemplate() == null) { throw new BadRequestException("Provided archive is faulty"); } @@ -335,9 +360,6 @@ public class DossierTemplateImportService { updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId()); dossierTemplateRepository.save(existingDossierTemplate); - // set rules - setRulesWhenCompiled(request, dossierTemplateId); - existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(existingDossierTemplate) .name())); @@ -448,12 +470,11 @@ public class DossierTemplateImportService { dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); dossierTemplateEntity.setCreatedBy(request.getUserId()); //set rules - setRulesWhenCompiled(request, dossierTemplateEntity.getId()); var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity); loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate)); - dossierTemplateId = loadedDossierTemplate.getId(); + dossierTemplateId = loadedDossierTemplate.getId(); // set colors this.setColors(dossierTemplateId, request.getColors()); @@ -527,17 +548,44 @@ public class DossierTemplateImportService { FileAttributesGeneralConfigurationEntity.class)); } + setRulesWhenCompiled(request, dossierTemplateId); + setComponentMappings(dossierTemplateId, request.getComponentMappings()); + long elapsedTime = System.currentTimeMillis() - start; log.info("stop import dossier template elapsedTime: " + elapsedTime + "for: " + dossierTemplateId); return dossierTemplateId; } + @SneakyThrows + private void setComponentMappings(String dossierTemplateId, List componentMappings) { + + List existingMappings = componentMappingService.getMetaDataByDossierTemplateId(dossierTemplateId); + existingMappings.forEach(metadata -> componentMappingService.delete(dossierTemplateId, metadata.getId())); + + for (ComponentMappingImportModel componentMapping : componentMappings) { + File tmpFile = Files.createTempFile("mapping", ".csv").toFile(); + try (var out = new FileOutputStream(tmpFile)) { + out.write(componentMapping.csvData()); + } + componentMappingService.create(dossierTemplateId, + componentMapping.metadata().getName(), + componentMapping.metadata().getFileName(), + componentMapping.metadata().getDelimiter(), + componentMapping.metadata().getEncoding(), + tmpFile); + assert tmpFile.delete(); + } + + } + + private void setRulesWhenCompiled(ImportTemplateResult request, String dossierTemplateId) { DroolsValidation droolsValidation = rulesValidationService.validateRules(RuleFileType.ENTITY, request.getRuleSet()); if (!droolsValidation.isCompiled()) { - droolsValidation.getSyntaxErrorMessages().forEach(errorMessage -> log.error(errorMessage.getMessage())); + droolsValidation.getSyntaxErrorMessages() + .forEach(errorMessage -> log.error(errorMessage.getMessage())); droolsValidation.getBlacklistErrorMessages() .forEach(errorMessage -> log.error(errorMessage.getMessage())); throw new BadRequestException("The entity rules do not compile or contain blacklisted keywords!"); @@ -546,7 +594,8 @@ public class DossierTemplateImportService { if (request.getComponentRuleSet() != null) { DroolsValidation componentDroolsValidation = rulesValidationService.validateRules(RuleFileType.COMPONENT, request.getComponentRuleSet()); if (!componentDroolsValidation.isCompiled()) { - componentDroolsValidation.getSyntaxErrorMessages().forEach(errorMessage -> log.error(errorMessage.getMessage())); + componentDroolsValidation.getSyntaxErrorMessages() + .forEach(errorMessage -> log.error(errorMessage.getMessage())); componentDroolsValidation.getBlacklistErrorMessages() .forEach(errorMessage -> log.error(errorMessage.getMessage())); throw new BadRequestException("The component rules do not compile or contain blacklisted keywords!"); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java index a60f18cd5..ba3903fa3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java @@ -89,7 +89,7 @@ public class DownloadService { request.getReportTemplateIds(), request.getRedactionPreviewColor()); websocketService.sendDownloadEvent(storageId, request.getUserId(), DownloadStatusValue.QUEUED); - addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).includeUnprocessed(request.getIncludeUnprocessed()).build(), 1); + addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).build(), 1); return new JSONPrimitive<>(storageId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java index 4dae625a7..79e57a923 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java @@ -66,6 +66,7 @@ public class FileStatusMapper { .lastIndexed(status.getLastIndexed()) .fileSize(status.getFileSize()) .fileErrorInfo(status.getFileErrorInfo()) + .componentMappingVersions(status.getComponentMappingVersions()) .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index e8a48209b..6f39faf7d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; import org.apache.commons.lang3.StringUtils; import org.springframework.retry.support.RetryTemplate; +import org.springframework.scheduling.annotation.Async; import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -39,17 +40,11 @@ public class FileStatusProcessingUpdateService { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); switch (analyzeResult.getMessageType()) { - case SURROUNDING_TEXT_ANALYSIS: break; - case ANALYSE: - case REANALYSE: default: - retryTemplate.execute(retryContext -> { - log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount()); - fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult); - return null; - }); + log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, 0); + fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult); if (!analyzeResult.isWasReanalyzed()) { indexingService.addToIndexingQueue(IndexMessageType.INSERT, dossier.getDossierTemplateId(), dossierId, fileId, 2); @@ -60,6 +55,7 @@ public class FileStatusProcessingUpdateService { if (analyzeResult.getAddedFileAttributes() != null && !analyzeResult.getAddedFileAttributes().isEmpty()) { fileStatusPersistenceService.addFileAttributes(dossierId, fileId, analyzeResult.getAddedFileAttributes()); } + break; } } @@ -79,16 +75,16 @@ public class FileStatusProcessingUpdateService { retryTemplate.execute(retryContext -> { log.info("Preprocessing dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount()); fileStatusService.setStatusPreProcessing(fileId, - fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); + fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); return null; }); var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId); if (updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()) { throw new ConflictException(String.format("Max Processing Retries exhausted for dossier %s and file %s with errorCount: %s", - dossierId, - fileId, - updatedFileEntity.getProcessingErrorCounter())); + dossierId, + fileId, + updatedFileEntity.getProcessingErrorCounter())); } } @@ -106,10 +102,10 @@ public class FileStatusProcessingUpdateService { retryTemplate.execute(retryContext -> { log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", - retryContext.getRetryCount(), - fileId, - dossierId, - fileErrorInfo != null ? fileErrorInfo.getCause() : null); + retryContext.getRetryCount(), + fileId, + dossierId, + fileErrorInfo != null ? fileErrorInfo.getCause() : null); fileStatusService.setStatusError(dossierId, fileId, fileErrorInfo); return null; }); @@ -123,7 +119,7 @@ public class FileStatusProcessingUpdateService { ocrFailed(dossierId, fileId, fileErrorInfo); } else { fileStatusService.setStatusOcrProcessing(fileId, - fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); + fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); fileStatusService.addToOcrQueue(dossierId, fileId, 2); } } @@ -136,7 +132,7 @@ public class FileStatusProcessingUpdateService { retryTemplate.execute(retryContext -> { log.info("Ocr processing dossier {} and file {}, Attempt to update status: {}", fileEntity.getDossierId(), fileId, retryContext.getRetryCount()); fileStatusService.setStatusOcrProcessing(fileId, - fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); + fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); return null; }); } @@ -149,9 +145,9 @@ public class FileStatusProcessingUpdateService { var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId); if (updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()) { throw new ConflictException(String.format("Max Ocr Retries exhausted for dossier %s and file %s with errorCount: %s", - updatedFileEntity.getDossierId(), - fileId, - updatedFileEntity.getProcessingErrorCounter())); + updatedFileEntity.getDossierId(), + fileId, + updatedFileEntity.getProcessingErrorCounter())); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 9818fefee..0440de55f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; + import org.apache.commons.lang3.StringUtils; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; @@ -90,6 +91,8 @@ public class FileStatusService { ViewedPagesPersistenceService viewedPagesPersistenceService; FileManagementServiceSettings fileManagementServiceSettings; LayoutParsingRequestFactory layoutParsingRequestFactory; + ComponentMappingService componentMappingService; + WebsocketService websocketService; @Transactional @@ -252,6 +255,7 @@ public class FileStatusService { .sectionsToReanalyse(sectionsToReanalyse) .fileId(fileId) .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted())) + .componentMappings(componentMappingService.getMetaDataByDossierTemplateId(dossierTemplate.getId())) .dossierTemplateId(dossier.getDossierTemplateId()) .lastProcessed(fileModel.getLastProcessed()) .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) @@ -517,7 +521,8 @@ public class FileStatusService { analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(), - analyzeResult.getAnalysisNumber()); + analyzeResult.getAnalysisNumber(), + analyzeResult.getUsedComponentMappings()); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java index 453f1e7ea..0857d3115 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java @@ -7,8 +7,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.service. import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; @@ -19,23 +22,28 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ReanalysisRequiredStatusService { - private final DictionaryPersistenceService dictionaryPersistenceService; - private final RulesPersistenceService rulesPersistenceService; - private final DossierPersistenceService dossierPersistenceService; - private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + DictionaryPersistenceService dictionaryPersistenceService; + RulesPersistenceService rulesPersistenceService; + DossierPersistenceService dossierPersistenceService; + LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + ComponentMappingService componentMappingService; public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) { @@ -78,8 +86,12 @@ public class ReanalysisRequiredStatusService { fileStatus.setDossierTemplateId(dossier.getDossierTemplateId()); fileStatus.setDossierStatusId(dossier.getDossierStatusId()); + if (fileStatus.isSoftOrHardDeleted()) { + log.debug("File {} is deleted, thus analysis is not required", fileStatus.getId()); + return new AnalysisRequiredResult(false, false); + } if (dossier.getSoftDeletedTime() != null || dossier.getHardDeletedTime() != null || dossier.getArchivedTime() != null) { - log.info("Dossier {} is deleted, thus analysis is not required", fileStatus.getDossierId()); + log.debug("Dossier {} is deleted, thus analysis is not required", fileStatus.getDossierId()); return new AnalysisRequiredResult(false, false); } @@ -118,39 +130,141 @@ public class ReanalysisRequiredStatusService { Map dossierVersionMap) { // get relevant versions - var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), k -> buildVersionData(dossier.getDossierTemplateId())); - var dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId())); + Map dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), + k -> buildVersionData(dossier.getDossierTemplateId())); + + Map componentMappingVersions = componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId()) + .stream() + .collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion)); + + Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId())); // compute matches + var mappingVersionAllMatch = mappingVersionsEqual(fileStatus, componentMappingVersions); var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L); var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L); var dictionaryVersionMatches = fileStatus.getDictionaryVersion() == dossierTemplateVersions.getOrDefault(DICTIONARY, -1L); var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L); var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion; - var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches; + var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch; var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches; + if (reanalysisRequired || fullAnalysisRequired) { - log.info( - "For file: {}-{} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}", - fileStatus.getId(), - fileStatus.getFilename(), - fileStatus.getRulesVersion(), - dossierTemplateVersions.getOrDefault(RULES, -1L), - fileStatus.getComponentRulesVersion(), - dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L), - fileStatus.getDictionaryVersion(), - dossierTemplateVersions.getOrDefault(DICTIONARY, -1L), - fileStatus.getLegalBasisVersion(), - dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L), - Math.max(fileStatus.getDossierDictionaryVersion(), 0), - dossierDictionaryVersion); + + log.info(buildMessage(fileStatus, + rulesVersionMatches, + dossierTemplateVersions, + componentRulesVersionMatches, + dictionaryVersionMatches, + legalBasisVersionMatches, + dossierDictionaryVersionMatches, + dossierDictionaryVersion, + mappingVersionAllMatch, + componentMappingVersions)); } return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired); } + private String buildMessage(FileModel fileStatus, + boolean rulesVersionMatches, + Map dossierTemplateVersions, + boolean componentRulesVersionMatches, + boolean dictionaryVersionMatches, + boolean legalBasisVersionMatches, + boolean dossierDictionaryVersionMatches, + Long dossierDictionaryVersion, + boolean mappingVersionAllMatch, + Map componentMappingVersions) { + + StringBuilder messageBuilder = new StringBuilder(); + messageBuilder.append("For file: ").append(fileStatus.getId()).append("-").append(fileStatus.getFilename()).append(" analysis is required because -> "); + + boolean needComma = false; + + if (!rulesVersionMatches) { + messageBuilder.append("ruleVersions: ").append(fileStatus.getRulesVersion()).append("/").append(dossierTemplateVersions.getOrDefault(RULES, -1L)); + needComma = true; + } + + if (!componentRulesVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("componentRuleVersions: ") + .append(fileStatus.getComponentRulesVersion()) + .append("/") + .append(dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L)); + needComma = true; + } + + if (!dictionaryVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("dictionaryVersions: ").append(fileStatus.getDictionaryVersion()).append("/").append(dossierTemplateVersions.getOrDefault(DICTIONARY, -1L)); + needComma = true; + } + + if (!legalBasisVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("legalBasisVersions: ").append(fileStatus.getLegalBasisVersion()).append("/").append(dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L)); + needComma = true; + } + + if (!dossierDictionaryVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("dossierDictionaryVersions: ").append(Math.max(fileStatus.getDossierDictionaryVersion(), 0)).append("/").append(dossierDictionaryVersion); + needComma = true; + } + + if (!mappingVersionAllMatch) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("componentMappingVersions: ").append(buildMappingVersionMatchesString(fileStatus.getComponentMappingVersions(), componentMappingVersions)); + needComma = true; + } + + return messageBuilder.toString(); + } + + + private String buildMappingVersionMatchesString(Map fileComponentMappingVersions, Map dbComponentMappingVersions) { + + Set allMappingNames = new HashSet<>(); + allMappingNames.addAll(fileComponentMappingVersions.keySet()); + allMappingNames.addAll(dbComponentMappingVersions.keySet()); + + StringBuilder sb = new StringBuilder(); + allMappingNames.stream() + .sorted() + .forEach(mappingName -> { + long fileVersion = fileComponentMappingVersions.getOrDefault(mappingName, -1); + long dbVersion = dbComponentMappingVersions.getOrDefault(mappingName, -1); + if (fileVersion != dbVersion) { + sb.append(mappingName).append(": ").append(fileVersion).append("/").append(dbVersion); + } + }); + return sb.toString(); + } + + + private static boolean mappingVersionsEqual(FileModel fileStatus, Map componentMappingVersions) { + + return fileStatus.getComponentMappingVersions().keySet().equals(componentMappingVersions.keySet()) && fileStatus.getComponentMappingVersions().keySet() + .stream() + .allMatch(name -> fileStatus.getComponentMappingVersions() + .get(name).equals(componentMappingVersions.get(name))); + } + + private Map buildVersionData(String dossierTemplateId) { var versions = new HashMap(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java index 166c86fa7..03d332582 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java @@ -4,16 +4,20 @@ import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicC import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; +import org.springframework.util.FileSystemUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -24,8 +28,10 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping; import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; @@ -43,6 +49,7 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.FileAttributesGeneralConfiguration; @@ -79,6 +86,7 @@ public class DossierTemplateExportService { private final RulesPersistenceService rulesPersistenceService; private final FileManagementStorageService fileManagementStorageService; private final ReportTemplatePersistenceService reportTemplatePersistenceService; + private final ComponentMappingService componentMappingService; private final ColorsService colorsService; private final EntryPersistenceService entryPersistenceService; private final ObjectMapper objectMapper = new ObjectMapper(); @@ -117,6 +125,7 @@ public class DossierTemplateExportService { objectMapper.registerModule(new JavaTimeModule()); DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId()); + downloadStatusPersistenceService.updateStatus(downloadJob.getStorageId(), DownloadStatusValue.GENERATING); String dossierTemplateId = extractDossierTemplateId(downloadStatus.getFilename()); var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId); @@ -234,6 +243,26 @@ public class DossierTemplateExportService { writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(), getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT)); } + Path mappingDir = Files.createTempDirectory("mappings"); + + List componentMappings = componentMappingService.getMappingFilesByDossierTemplateId(dossierTemplateId, mappingDir); + + List metadata = componentMappings.stream() + .map(ComponentMapping::metaData) + .toList(); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename(), + ExportFilename.COMPONENT_MAPPINGS_FILE.getFilename() + JSON_EXT, + objectMapper.writeValueAsBytes(metadata))); + for (ComponentMapping componentMapping : componentMappings) { + try (var in = new FileInputStream(componentMapping.file())) { + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename(), + componentMapping.metaData().getName() + ".csv", + in.readAllBytes())); + } + } + + FileSystemUtils.deleteRecursively(mappingDir); + storeZipFile(downloadStatus.getStorageId(), fileSystemBackedArchiver); downloadStatusPersistenceService.updateStatus(downloadStatus, DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index ebc985b83..970b1a173 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.job; import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -69,7 +70,9 @@ public class AutomaticAnalysisJob implements Job { if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) { log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId()); return; - }var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); + } + + var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); if (redactionQueueInfo != null) { log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", tenant.getTenantId(), @@ -80,17 +83,17 @@ public class AutomaticAnalysisJob implements Job { var consumerCount = redactionQueueInfo.getConsumerCount(); if (redactionQueueInfo.getMessageCount() <= consumerCount * 5) { // queue up 5 files - var allStatuses = getAllRelevantStatuses(); + List allStatuses = getAllRelevantStatuses(); allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated)); - var allStatusesIterator = allStatuses.iterator(); + Iterator allStatusesIterator = allStatuses.iterator(); log.debug("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size()); - var queuedFiles = 0; + int queuedFiles = 0; while (queuedFiles < (consumerCount * 5) && allStatusesIterator.hasNext()) { - var next = allStatusesIterator.next(); + FileModel next = allStatusesIterator.next(); // in case the file doesn't have numberOfPages set, we assume an average. reanalyseFile(next); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index d47301caa..16b45a1c9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -2,9 +2,12 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -14,6 +17,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier; @@ -23,10 +27,12 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Websock import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; +import jakarta.persistence.EntityManager; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -42,6 +48,7 @@ public class FileStatusPersistenceService { private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; private final DossierPersistenceService dossierService; private final WebsocketService websocketService; + private final EntityManager entityManager; public void createStatus(String dossierId, String fileId, String filename, String uploader, long size) { @@ -61,6 +68,7 @@ public class FileStatusPersistenceService { file.setFileManipulationDate(now); file.setProcessingErrorCounter(0); file.setFileSize(size); + file.setComponentMappingVersions(new ArrayList<>()); fileRepository.save(file); } @@ -91,21 +99,15 @@ public class FileStatusPersistenceService { private int calculateProcessingErrorCounter(String fileId, ProcessingStatus processingStatus) { - switch (processingStatus) { - case ERROR: - return fileRepository.findById(fileId) - .map(FileEntity::getProcessingErrorCounter) - .orElse(0) + 1; - - case PROCESSED: - case REPROCESS: - return 0; - - default: - return fileRepository.findById(fileId) - .map(FileEntity::getProcessingErrorCounter) - .orElse(0); - } + return switch (processingStatus) { + case ERROR -> fileRepository.findById(fileId) + .map(FileEntity::getProcessingErrorCounter) + .orElse(0) + 1; + case PROCESSED, REPROCESS -> 0; + default -> fileRepository.findById(fileId) + .map(FileEntity::getProcessingErrorCounter) + .orElse(0); + }; } @@ -121,11 +123,13 @@ public class FileStatusPersistenceService { long duration, long dossierDictionaryVersion, int analysisVersion, - int analysisNumber) { + int analysisNumber, + List usedComponentMappings) { if (isFileDeleted(fileId)) { return; } + fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, @@ -141,10 +145,30 @@ public class FileStatusPersistenceService { analysisNumber, calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED)); + // must be modifiable, otherwise hibernate fails + List versionEntities = getFileEntityComponentMappingVersionEntities(usedComponentMappings); + + FileEntity fileEntity = entityManager.find(FileEntity.class, fileId); + + fileEntity.setComponentMappingVersions(versionEntities); + + entityManager.merge(fileEntity); + websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.FINISHED, analysisNumber); } + private static List getFileEntityComponentMappingVersionEntities(List usedComponentMappings) { + + if (usedComponentMappings == null) { + return Collections.emptyList(); + } + return usedComponentMappings.stream() + .map(cm -> new FileEntityComponentMappingVersionEntity(cm.getName(), cm.getVersion())) + .collect(Collectors.toList()); + } + + @Transactional public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ComponentMappingRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ComponentMappingRepository.java new file mode 100644 index 000000000..219d6866d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ComponentMappingRepository.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; + +import java.util.List; +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; + +import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DigitalSignatureEntity; + +public interface ComponentMappingRepository extends JpaRepository { + + List findByDossierTemplateId(String dossierTemplateId); + + + boolean existsByNameAndDossierTemplateId(String name, String dossierTemplateId); + + + Optional findByIdAndDossierTemplateId(String id, String dossierTemplateId); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 775564d99..6379bbce2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -47,12 +47,21 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " - + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, " - + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " - + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " - + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " - + "where f.id = :fileId") + @Query(""" + update FileEntity f set f.numberOfPages = :numberOfPages, \ + f.processingStatus = :processingStatus, \ + f.dictionaryVersion = :dictionaryVersion, \ + f.rulesVersion = :rulesVersion, \ + f.componentRulesVersion = :componentRulesVersion, \ + f.legalBasisVersion = :legalBasisVersion, \ + f.analysisDuration = :analysisDuration, \ + f.dossierDictionaryVersion = :dossierDictionaryVersion, \ + f.analysisVersion = :analysisVersion, \ + f.numberOfAnalyses = :analysisNumber, \ + f.lastUpdated = :lastUpdated, \ + f.lastProcessed = :lastProcessed, \ + f.processingErrorCounter = :processingErrorCounter \ + where f.id = :fileId""") void updateProcessingStatus(@Param("fileId") String fileId, @Param("numberOfPages") int numberOfPages, @Param("processingStatus") ProcessingStatus processingStatus, @@ -168,12 +177,13 @@ public interface FileRepository extends JpaRepository { @Param("lastUpdated") OffsetDateTime lastUpdated, @Param("softDeletedTime") OffsetDateTime softDeletedTime); + @Modifying @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id in (:fileIds)") int softDeleteFiles(@Param("fileIds") List fileIds, - @Param("processingStatus") ProcessingStatus processingStatus, - @Param("lastUpdated") OffsetDateTime lastUpdated, - @Param("softDeletedTime") OffsetDateTime softDeletedTime); + @Param("processingStatus") ProcessingStatus processingStatus, + @Param("lastUpdated") OffsetDateTime lastUpdated, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); @Modifying @@ -380,10 +390,7 @@ public interface FileRepository extends JpaRepository { + " when f.deleted is not null then f.deleted " + "end " + "where f.id in (:fileIds)") - int hardDeleteFiles(@Param("fileIds") List fileIds, - @Param("processingStatus") ProcessingStatus processingStatus, - @Param("deletionTime") OffsetDateTime deletionTime); - + int hardDeleteFiles(@Param("fileIds") List fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("deletionTime") OffsetDateTime deletionTime); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java index 7730046c4..ab16ad7f1 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java @@ -53,7 +53,7 @@ public class OCRProcessingMessageReceiver { response.getNumberOfOCRedPages()); } - log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE); + log.debug("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE); } @@ -63,7 +63,7 @@ public class OCRProcessingMessageReceiver { var response = objectMapper.readValue(failedMessage.getBody(), OCRStatusUpdateResponse.class); - log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL); + log.debug("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java index 8f6fbc97a..c1d89f926 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java @@ -1,7 +1,12 @@ package com.iqser.red.service.persistence.management.v1.processor.utils; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; @@ -14,6 +19,18 @@ public class FileModelMapper implements BiConsumer { fileEntity.getFileAttributes() .forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue())); fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp())); + fileModel.setComponentMappingVersions(getComponentMappingVersions(fileEntity)); + } + + + private static Map getComponentMappingVersions(FileEntity fileEntity) { + + if (Objects.isNull(fileEntity.getComponentMappingVersions())) { + return Collections.emptyMap(); + } + return fileEntity.getComponentMappingVersions() + .stream() + .collect(Collectors.toMap(FileEntityComponentMappingVersionEntity::getName, FileEntityComponentMappingVersionEntity::getVersion)); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index f01c902e0..470f13643 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -202,4 +202,8 @@ databaseChangeLog: - include: file: db/changelog/tenant/125-add-max-size-for-legal-basis-in-manual-legal-basis-change.yaml - include: - file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml \ No newline at end of file + file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml + - include: + file: db/changelog/tenant/127-add-component-mapping-table.yaml + - include: + file: db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/127-add-component-mapping-table.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/127-add-component-mapping-table.yaml new file mode 100644 index 000000000..e9679364e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/127-add-component-mapping-table.yaml @@ -0,0 +1,97 @@ +databaseChangeLog: + - changeSet: + id: create-table-component_mappings + author: kilian + changes: + - createTable: + tableName: component_mappings + columns: + - column: + name: id + type: VARCHAR(255) + constraints: + primaryKey: true + - column: + name: dossier_template_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: storage_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: name + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: file_name + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: version + type: integer + constraints: + nullable: false + defaultValue: '-1' + - column: + name: changed_date + type: TIMESTAMP WITH TIME ZONE + constraints: + nullable: false + - column: + name: number_of_lines + type: INTEGER + constraints: + nullable: false + defaultValue: '0' + - column: + name: encoding + type: VARCHAR(255) + constraints: + nullable: false + defaultValue: 'UTF-8' + - column: + name: delimiter + type: CHAR(1) + constraints: + nullable: false + defaultValue: ',' + - addUniqueConstraint: + columnNames: name, dossier_template_id + constraintName: unique_name_dossier_template + tableName: component_mappings + - addForeignKeyConstraint: + baseColumnNames: dossier_template_id + baseTableName: component_mappings + constraintName: fk_entity_dossier_template_id + referencedColumnNames: id + referencedTableName: dossier_template + onDelete: CASCADE + onUpdate: CASCADE + + - createTable: + tableName: component_mapping_column_labels + columns: + - column: + name: component_mapping_entity_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: column_labels + type: VARCHAR(255) + constraints: + nullable: false + + - addForeignKeyConstraint: + baseColumnNames: component_mapping_entity_id + baseTableName: component_mapping_column_labels + constraintName: fk_component_mapping_column_labels_entity_id + referencedColumnNames: id + referencedTableName: component_mappings + onDelete: CASCADE + onUpdate: CASCADE diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml new file mode 100644 index 000000000..e9dbcefa2 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml @@ -0,0 +1,31 @@ +databaseChangeLog: + - changeSet: + id: add_component_mapping_version_to_file_entity + author: kilian + changes: + - createTable: + tableName: file_entity_component_mapping_versions + columns: + - column: + name: file_entity_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: name + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: version + type: INT + constraints: + nullable: false + - addForeignKeyConstraint: + baseColumnNames: file_entity_id + baseTableName: file_entity_component_mapping_versions + constraintName: fk_component_mapping_version_entity_file + referencedColumnNames: id + referencedTableName: file + onDelete: CASCADE + onUpdate: CASCADE diff --git a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts index 19ed8474c..6e14c8152 100644 --- a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts @@ -27,6 +27,7 @@ dependencies { api("junit:junit:4.13.2") api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0") api("net.logstash.logback:logstash-logback-encoder:7.4") + testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2") testImplementation("org.springframework.security:spring-security-test:6.0.2") testImplementation("org.testcontainers:postgresql:1.17.1") diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index 92332f560..175e5ee92 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import org.apache.commons.codec.binary.Base64; @@ -115,6 +116,7 @@ public class FileTesterAndProvider { result.setDossierDictionaryVersion(1); result.setAnalysisNumber(1); result.setAnalysisVersion(1); + result.setUsedComponentMappings(Collections.emptyList()); fileStatusService.setStatusSuccessful(dossierId, fileId, result); fileStatusService.setStatusProcessed(fileId); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/CustomPermissionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/CustomPermissionTest.java index 38f5d7d97..50bacb882 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/CustomPermissionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/CustomPermissionTest.java @@ -10,10 +10,6 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import com.iqser.red.service.peristence.v1.server.integration.client.CustomPermissionClient; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java index cc9497ce1..19d4f8ced 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java @@ -26,6 +26,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService; @@ -73,6 +74,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { @MockBean private ColorsService colorsService; @MockBean + private ComponentMappingService componentMappingService; + @MockBean private StorageService storageService; @MockBean private DossierStatusPersistenceService dossierStatusPersistenceService; @@ -125,7 +128,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { storageService, dossierStatusPersistenceService, watermarkService, - fileManagementStorageService); + fileManagementStorageService, + componentMappingService); dossierTemplateExportService = new DossierTemplateExportService(dossierTemplatePersistenceService, downloadStatusPersistenceService, dossierAttributeConfigPersistenceService, @@ -136,6 +140,7 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { rulesPersistenceService, fileManagementStorageService, reportTemplatePersistenceService, + componentMappingService, colorsService, entryPersistenceService, watermarkService, diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneServiceTest.java index fd9846deb..d783fd0fd 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneServiceTest.java @@ -28,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService; @@ -91,6 +92,8 @@ public class DossierTemplateCloneServiceTest { private FileManagementStorageService fileManagementStorageService; @MockBean private TypeRepository typeRepository; + @MockBean + private ComponentMappingService componentMappingService; private DossierTemplateCloneService dossierTemplateCloneService; @@ -124,7 +127,8 @@ public class DossierTemplateCloneServiceTest { storageService, dossierStatusPersistenceService, watermarkService, - fileManagementStorageService); + fileManagementStorageService, + componentMappingService); dummyTemplate = new DossierTemplateEntity(); setNonDefaultValues(dummyTemplate); @@ -169,7 +173,7 @@ public class DossierTemplateCloneServiceTest { if (getterMethod != null) { value = getterMethod.invoke(obj); - } else if (alternativeGetterMethod != null){ + } else if (alternativeGetterMethod != null) { value = alternativeGetterMethod.invoke(obj); } else { return; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java new file mode 100644 index 000000000..4ca22ffc9 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java @@ -0,0 +1,173 @@ +package com.iqser.red.service.peristence.v1.server.integration.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.util.FileSystemUtils; + +import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; +import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService; +import com.iqser.red.service.persistence.management.v1.processor.service.export.DossierTemplateExportService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.SneakyThrows; + +public class DossierTemplateImportExportTest extends AbstractPersistenceServerServiceTest { + + public static final String IMPORTED_TEMPLATE_NAME = "imported-template"; + public static final String USER_ID = "NutzerIdentifikationsNummer"; + public static final String AFTER = "after"; + public static final String BEFORE = "before"; + + @Autowired + DossierTemplateImportService dossierTemplateImportService; + + @Autowired + DossierTemplateExportService dossierTemplateExportService; + + @Autowired + StorageService storageService; + + @MockBean + DownloadStatusPersistenceService downloadStatusPersistenceService; + + + @AfterEach + public void clearStorage() { + + ((FileSystemBackedStorageService) storageService).clearStorage(); + } + + + @Test + @SneakyThrows +// @DirtiesContext + public void testImportExportRoundtrip() { + + TenantContext.setTenantId(TENANT_1); + Path outDir = Files.createTempDirectory(IMPORTED_TEMPLATE_NAME); + + Path dossierTemplateExportArchive = new ClassPathResource("files/dossiertemplates/DossierTemplate.zip").getFile().toPath(); + + String importedDossierTemplateId = dossierTemplateImportService.importDossierTemplate(ImportDossierTemplateRequest.builder() + .archive(Files.readAllBytes(dossierTemplateExportArchive)) + .userId(USER_ID) + .build()); + + when(downloadStatusPersistenceService.getStatus(anyString()))// + .thenReturn(DownloadStatusEntity.builder().filename(importedDossierTemplateId + ".zip").storageId(IMPORTED_TEMPLATE_NAME).build()); + + dossierTemplateExportService.createDownloadArchive(DownloadJob.builder().storageId(importedDossierTemplateId).userId(USER_ID).build()); + + File tmpFile = Files.createTempFile(IMPORTED_TEMPLATE_NAME, ".zip").toFile(); + storageService.downloadTo(TenantContext.getTenantId(), IMPORTED_TEMPLATE_NAME, tmpFile); + + unzip(dossierTemplateExportArchive.toFile().toString(), outDir.resolve(BEFORE)); + unzip(tmpFile.toString(), outDir.resolve(AFTER)); + + assert tmpFile.delete(); + + Map beforeContents = getDirectoryContents(outDir.resolve(BEFORE)); + Map afterContents = getDirectoryContents(outDir.resolve(AFTER)); + + assertEquals(beforeContents.size(), afterContents.size()); + assertEquals(beforeContents.keySet(), afterContents.keySet()); + + // can't assert equality on contents as UUID's are supposed to change on import + + FileSystemUtils.deleteRecursively(outDir); + } + + + public static void unzip(String zipFilePath, Path destDirectory) throws IOException { + + File destDir = destDirectory.toFile(); + if (!destDir.exists()) { + destDir.mkdir(); + } + try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath))) { + ZipEntry entry = zipIn.getNextEntry(); + while (entry != null) { + String filePath = destDirectory + File.separator + entry.getName(); + if (!entry.isDirectory()) { + // if the entry is a file, extracts it + extractFile(zipIn, filePath); + } else { + // if the entry is a directory, make the directory + File dir = new File(filePath); + dir.mkdirs(); + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); + } + } + } + + + private static void extractFile(ZipInputStream zipIn, String filePath) throws IOException { + + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) { + byte[] bytesIn = new byte[4096]; + int read; + while ((read = zipIn.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + } + } + + + private static Map getDirectoryContents(Path dir) throws IOException, NoSuchAlgorithmException { + + Map contents = new HashMap<>(); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + + Files.walkFileTree(dir, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + + contents.put(dir.relativize(file), digest.digest(Files.readAllBytes(file))); + return FileVisitResult.CONTINUE; + } + + + @Override + public FileVisitResult preVisitDirectory(Path subDir, BasicFileAttributes attrs) { + + contents.put(dir.relativize(subDir), new byte[0]); + return FileVisitResult.CONTINUE; + } + }); + + return contents; + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java index 55535290d..110b05765 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -298,7 +299,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k); if (k == 1) { - fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1); + fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1, Collections.emptyList()); reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k)))); } if (k == 2) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index 02f42c1fe..8836008f4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -585,7 +585,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { var status = statuses.getDownloadStatus() .iterator().next(); - exportDownloadReportMessageReceiver.receive(new DownloadJob(status.getUserId(), status.getStorageId(), false)); + exportDownloadReportMessageReceiver.receive(new DownloadJob(status.getUserId(), status.getStorageId())); // add new justifications legalBasisClient.setLegalBasisMapping(List.of(new LegalBasis("nameAgain", "description", "reason")), dossierTemplate.getId()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadTest.java index 0fc49111c..6d1597bc8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadTest.java @@ -99,7 +99,7 @@ public class DownloadTest extends AbstractPersistenceServerServiceTest { .fileIds(List.of(file2.getId())) .build()); - downloadMessageReceiver.receive(new DownloadJob(userProvider.getUserId(), downloads.getStorageId(), false)); + downloadMessageReceiver.receive(new DownloadJob(userProvider.getUserId(), downloads.getStorageId())); var reportInfoId = downloads.getStorageId().substring(0, downloads.getStorageId().length() - 3) + "/REPORT_INFO.json"; storageService.storeJSONObject(TenantContext.getTenantId(), reportInfoId, new ArrayList<>()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index 8f9d9a87b..ce27b8a53 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -9,14 +9,17 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository; + import org.assertj.core.util.Lists; import org.bson.BsonArray; import org.bson.BsonDocument; import org.bson.BsonString; +import org.checkerframework.checker.units.qual.A; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; @@ -38,6 +41,7 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; +import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.StatementCallback; import org.springframework.jdbc.datasource.SingleConnectionDataSource; @@ -50,8 +54,10 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.provisioning.InMemoryUserDetailsManager; import org.springframework.security.web.SecurityFilterChain; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.support.TestPropertySourceUtils; import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.peristence.v1.server.Application; @@ -138,6 +144,9 @@ import lombok.extern.slf4j.Slf4j; @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = "spring-hibernate-query-utils.n-plus-one-queries-detection.error-level=INFO") public abstract class AbstractPersistenceServerServiceTest { + public static final String TENANT_1 = "redaction"; + public static final String TENANT_2 = "redaction2"; + @MockBean protected Scheduler scheduler; @MockBean @@ -273,7 +282,7 @@ public abstract class AbstractPersistenceServerServiceTest { @BeforeEach public void setupTenantContext() { - TenantContext.setTenantId("redaction"); + TenantContext.setTenantId(TENANT_1); } @@ -289,7 +298,7 @@ public abstract class AbstractPersistenceServerServiceTest { createTenants(); - TenantContext.setTenantId("redaction"); + TenantContext.setTenantId(TENANT_1); ApplicationConfig appConfig = ApplicationConfig.builder().downloadCleanupDownloadFilesHours(8).downloadCleanupNotDownloadFilesHours(72).softDeleteCleanupTime(96).build(); applicationConfigService.saveApplicationConfiguration(MagicConverter.convert(appConfig, ApplicationConfigurationEntity.class)); @@ -346,9 +355,9 @@ public abstract class AbstractPersistenceServerServiceTest { if (tenantsClient.getTenants() == null || tenantsClient.getTenants().isEmpty()) { var redactionTenant = new TenantResponse(); - redactionTenant.setTenantId("redaction"); - redactionTenant.setGuid("redaction"); - redactionTenant.setDisplayName("redaction"); + redactionTenant.setTenantId(TENANT_1); + redactionTenant.setGuid(TENANT_1); + redactionTenant.setDisplayName(TENANT_1); redactionTenant.setAuthDetails(new AuthDetails()); redactionTenant.setDatabaseConnection(DatabaseConnection.builder() .driver("postgresql") @@ -388,9 +397,9 @@ public abstract class AbstractPersistenceServerServiceTest { .build()); var redactionTenant2 = new TenantResponse(); - redactionTenant2.setTenantId("redaction2"); - redactionTenant2.setGuid("redaction2"); - redactionTenant2.setDisplayName("redaction2"); + redactionTenant2.setTenantId(TENANT_2); + redactionTenant2.setGuid(TENANT_2); + redactionTenant2.setDisplayName(TENANT_2); redactionTenant2.setAuthDetails(new AuthDetails()); redactionTenant2.setDatabaseConnection(DatabaseConnection.builder() .driver("postgresql") @@ -425,19 +434,19 @@ public abstract class AbstractPersistenceServerServiceTest { .username(MONGO_USERNAME) .password(encryptionDecryptionService.encrypt(MONGO_PASSWORD)) .address(mongoDbContainer.getHost() + ":" + mongoDbContainer.getFirstMappedPort()) - .database("redaction2") + .database(TENANT_2) .options("") .build()); - when(tenantsClient.getTenant("redaction")).thenReturn(redactionTenant); - when(tenantsClient.getTenant("redaction2")).thenReturn(redactionTenant2); + when(tenantsClient.getTenant(TENANT_1)).thenReturn(redactionTenant); + when(tenantsClient.getTenant(TENANT_2)).thenReturn(redactionTenant2); when(tenantsClient.getTenants()).thenReturn(List.of(redactionTenant, redactionTenant2)); try { - tenantCreatedListener.createTenant(new TenantCreatedEvent("redaction")); - tenantCreatedListener.createTenant(new TenantCreatedEvent("redaction2")); - mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent("redaction")); - mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent("redaction2")); + tenantCreatedListener.createTenant(new TenantCreatedEvent(TENANT_1)); + tenantCreatedListener.createTenant(new TenantCreatedEvent(TENANT_2)); + mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent(TENANT_1)); + mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent(TENANT_2)); } catch (Exception e) { e.printStackTrace(); @@ -515,7 +524,6 @@ public abstract class AbstractPersistenceServerServiceTest { notificationPreferencesRepository.deleteAll(); indexInformationRepository.deleteAll(); applicationConfigRepository.deleteAll(); - entityLogEntryDocumentRepository.deleteAll(); entityLogDocumentRepository.deleteAll(); @@ -526,8 +534,16 @@ public abstract class AbstractPersistenceServerServiceTest { @Slf4j static class Initializer implements ApplicationContextInitializer { + static AtomicInteger UniquePortFactory = new AtomicInteger(28081); + + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + // any time the WebEnvironment is re-instantiated use a new port, as it sometimes leads to PortBindExceptions + ConfigurableEnvironment environment = configurableApplicationContext.getEnvironment(); + int port = UniquePortFactory.getAndIncrement(); + TestPropertySourceUtils.addInlinedPropertiesToEnvironment(environment, "server.port=" + port); + var postgreSQLContainerMaster = SpringPostgreSQLTestContainer.getInstance().withDatabaseName("integration-tests-db-master").withUsername("sa").withPassword("sa"); postgreSQLContainerMaster.start(); @@ -537,8 +553,8 @@ public abstract class AbstractPersistenceServerServiceTest { var mongoInstance = MongoDBTestContainer.getInstance(); mongoInstance.start(); - createMongoDBDatabase(mongoInstance, "redaction"); - createMongoDBDatabase(mongoInstance, "redaction2"); + createMongoDBDatabase(mongoInstance, TENANT_1); + createMongoDBDatabase(mongoInstance, TENANT_2); log.info("Hosts are - Redis: {}, Postgres: {}, MongoDB: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost(), mongoInstance.getHost()); @@ -553,37 +569,36 @@ public abstract class AbstractPersistenceServerServiceTest { } - } + private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer, String databaseName) { - private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer, String databaseName) { + try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/", + MONGO_USERNAME, + MONGO_PASSWORD, + mongoDBTestContainer.getHost(), + mongoDBTestContainer.getFirstMappedPort()))) { + MongoDatabase database = mongoClient.getDatabase(databaseName); + BsonDocument createUserCommand = new BsonDocument(); + createUserCommand.append("createUser", new BsonString(MONGO_USERNAME)); + createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD)); + BsonArray roles = new BsonArray(); + roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(databaseName))); + createUserCommand.append("roles", roles); - try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/", - MONGO_USERNAME, - MONGO_PASSWORD, - mongoDBTestContainer.getHost(), - mongoDBTestContainer.getFirstMappedPort()))) { - MongoDatabase database = mongoClient.getDatabase(databaseName); - BsonDocument createUserCommand = new BsonDocument(); - createUserCommand.append("createUser", new BsonString(MONGO_USERNAME)); - createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD)); - BsonArray roles = new BsonArray(); - roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(databaseName))); - createUserCommand.append("roles", roles); + try { + database.runCommand(createUserCommand); + } catch (MongoCommandException mongoCommandException) { + // ignore user already exists + if (mongoCommandException.getErrorCode() != 51003) { + throw mongoCommandException; + } - try { - database.runCommand(createUserCommand); - } catch (MongoCommandException mongoCommandException) { - // ignore user already exists - if (mongoCommandException.getErrorCode() != 51003) { - throw mongoCommandException; } } - } - } + } @Configuration @EnableWebSecurity diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip new file mode 100644 index 000000000..bc076a829 Binary files /dev/null and b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip differ diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java index b00739042..29014cc47 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Set; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import lombok.AllArgsConstructor; import lombok.Builder; @@ -34,5 +35,8 @@ public class AnalyzeRequest { @Builder.Default private List fileAttributes = new ArrayList<>(); + @Builder.Default + private List componentMappings = new ArrayList<>(); + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java index 5ef8e7b29..f6148ec5b 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java @@ -1,8 +1,10 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; +import java.util.List; import java.util.Set; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import lombok.AllArgsConstructor; import lombok.Builder; @@ -37,4 +39,6 @@ public class AnalyzeResult { private Set addedFileAttributes; + private List usedComponentMappings; + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java index 1af0c17f7..d3baf9030 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java @@ -1,6 +1,7 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; import java.time.OffsetDateTime; +import java.util.Map; import java.util.Set; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; @@ -147,7 +148,8 @@ public class FileStatus { private OffsetDateTime lastIndexed; @Schema(description = "The error information for the error state of the file") private FileErrorInfo fileErrorInfo; - + @Schema(description = "Shows which versions of each mapping the last analysis has been performed") + private Map componentMappingVersions; @Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.") public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java new file mode 100644 index 000000000..410a74afd --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java @@ -0,0 +1,38 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.component; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ComponentMappingMetadata { + + String id; + + String name; + + String fileName; + + String storageId; + + Integer version; + + List columnLabels; + + Integer numberOfLines; + + String encoding; + + char delimiter; + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java index 06e45cee0..6794ba1aa 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java @@ -73,6 +73,7 @@ public class FileModel { private OffsetDateTime fileManipulationDate; private boolean hasHighlights; private FileErrorInfo fileErrorInfo; + private Map componentMappingVersions = new HashMap<>(); public long getFileSize() { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java index a2125b0c2..80112f08f 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java @@ -19,6 +19,5 @@ public enum ProcessingStatus { PRE_PROCESSED, FIGURE_DETECTION_ANALYZING, TABLE_PARSING_ANALYZING, - VISUAL_LAYOUT_PARSING_ANALYZING } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java new file mode 100644 index 000000000..1aa95b411 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport; + +import java.io.File; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; + +public record ComponentMappingImportModel(ComponentMappingMetadata metadata, byte[] csvData) { + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java index aff455da0..905b4fed0 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java @@ -19,7 +19,9 @@ public enum ExportFilename { DOSSIER_TYPE("dossierType"), ENTRIES("entries"), FALSE_POSITIVES("falsePositives"), - FALSE_RECOMMENDATION("falseRecommendations"); + FALSE_RECOMMENDATION("falseRecommendations"), + COMPONENT_MAPPINGS_FOLDER("mappings"), + COMPONENT_MAPPINGS_FILE("componentMappingsList"); @Getter private final String filename; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java index fe4a7e29a..6f20e791e 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java @@ -50,6 +50,9 @@ public class ImportTemplateResult { @Builder.Default public List fileAttributesConfigs = new ArrayList<>(); + @Builder.Default + public List componentMappings = new ArrayList<>(); + @Builder.Default public List legalBases = new ArrayList<>(); diff --git a/publish-custom-image.sh b/publish-custom-image.sh index b680eb16a..5a0def0ea 100755 --- a/publish-custom-image.sh +++ b/publish-custom-image.sh @@ -1,5 +1,9 @@ #!/bin/bash + +set -e + dir=${PWD##*/} + gradle assemble # Get the current Git branch @@ -11,5 +15,31 @@ commit_hash=$(git rev-parse --short=5 HEAD) # Combine branch and commit hash buildName="${USER}-${branch}-${commit_hash}" -gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName -echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName" +gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${buildName} + +newImageName="nexus.knecon.com:5001/red/${dir}-server-v1:${buildName}" + +echo "full image name:" +echo ${newImageName} +echo "" + +if [ -z "$1" ]; then + exit 0 +fi + +namespace=${1} +deployment_name="persistence-service-v1" + +echo "deploying to ${namespace}" + +oldImageName=$(rancher kubectl -n ${namespace} get deployment ${deployment_name} -o=jsonpath='{.spec.template.spec.containers[*].image}') + +if [ "${newImageName}" = "${oldImageName}" ]; then + echo "Image tag did not change, redeploying..." + rancher kubectl rollout restart deployment ${deployment_name} -n ${namespace} +else + echo "upgrading the image tag..." + rancher kubectl set image deployment/${deployment_name} ${deployment_name}=${newImageName} -n ${namespace} +fi +rancher kubectl rollout status deployment ${deployment_name} -n ${namespace} +echo "Built ${deployment_name}:${buildName} and deployed to ${namespace}"