Merge branch 'RED-9145' into 'master'
RED-9145: add component mappings Closes RED-9145 See merge request redactmanager/persistence-service!485
This commit is contained in:
Kilian Schüttler
commit
9e70d9e199
@ -65,6 +65,15 @@ java {
|
||||
}
|
||||
|
||||
allprojects {
|
||||
|
||||
tasks.withType<Javadoc> {
|
||||
options {
|
||||
this as StandardJavadocDocletOptions
|
||||
addBooleanOption("Xdoclint:none", true)
|
||||
addStringOption("Xmaxwarns", "1")
|
||||
}
|
||||
}
|
||||
|
||||
publishing {
|
||||
publications {
|
||||
create<MavenPublication>(name) {
|
||||
|
||||
@ -7,6 +7,9 @@ dependencies {
|
||||
api(project(":persistence-service-processor-v1"))
|
||||
api(project(":persistence-service-external-api-v2"))
|
||||
api(project(":persistence-service-external-api-impl-v1"))
|
||||
|
||||
implementation("org.mapstruct:mapstruct:1.5.5.Final")
|
||||
annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final")
|
||||
}
|
||||
|
||||
description = "persistence-service-external-api-impl-v2"
|
||||
|
||||
@ -12,10 +12,10 @@ import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierTemplateController;
|
||||
import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntityReference;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
|
||||
@ -37,10 +37,10 @@ import lombok.experimental.FieldDefaults;
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class ComponentControllerV2 implements ComponentResource {
|
||||
|
||||
DossierTemplateController dossierTemplateController;
|
||||
ComponentLogService componentLogService;
|
||||
StatusController statusController;
|
||||
FileStatusService fileStatusService;
|
||||
DossierTemplatePersistenceService dossierTemplatePersistenceService;
|
||||
|
||||
|
||||
@Override
|
||||
@ -49,7 +49,7 @@ public class ComponentControllerV2 implements ComponentResource {
|
||||
@PathVariable(FILE_ID_PARAM) String fileId,
|
||||
@RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) {
|
||||
|
||||
dossierTemplateController.getDossierTemplate(dossierTemplateId);
|
||||
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
|
||||
var componentLog = componentLogService.getComponentLog(dossierId, fileId, true);
|
||||
|
||||
Map<String, List<String>> basicComponent = new LinkedHashMap<>();
|
||||
@ -119,11 +119,12 @@ public class ComponentControllerV2 implements ComponentResource {
|
||||
@PathVariable(DOSSIER_ID_PARAM) String dossierId,
|
||||
@RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) {
|
||||
|
||||
dossierTemplateController.getDossierTemplate(dossierTemplateId);
|
||||
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
|
||||
var dossierFiles = statusController.getDossierStatus(dossierId);
|
||||
return new FileComponentsList(dossierFiles.stream()
|
||||
.map(file -> getComponents(dossierTemplateId, dossierId, file.getFileId(), includeDetails))
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -6,8 +6,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.WRITE_RULES;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
@ -23,11 +26,17 @@ import org.springframework.web.bind.annotation.RequestPart;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierTemplateController;
|
||||
import com.iqser.red.persistence.service.v1.external.api.impl.controller.FileAttributesController;
|
||||
import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMappingMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.RulesValidationMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
|
||||
@ -37,6 +46,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinition;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource;
|
||||
@ -46,21 +57,27 @@ import feign.FeignException;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "1. Dossier templates endpoints", description = "Provides operations related to dossier templates")
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class DossierTemplateControllerV2 implements DossierTemplateResource {
|
||||
|
||||
private static final String RULES_DOWNLOAD_FILE_NAME_SUFFIX = "-rules.drl";
|
||||
|
||||
private final DossierTemplateController dossierTemplateController;
|
||||
private final RulesPersistenceService rulesPersistenceService;
|
||||
private final RulesValidationService rulesValidationService;
|
||||
private final AuditPersistenceService auditPersistenceService;
|
||||
private final FileAttributesController fileAttributesController;
|
||||
DossierTemplateController dossierTemplateController;
|
||||
RulesPersistenceService rulesPersistenceService;
|
||||
RulesValidationService rulesValidationService;
|
||||
AuditPersistenceService auditPersistenceService;
|
||||
FileAttributesController fileAttributesController;
|
||||
ComponentMappingService componentMappingService;
|
||||
ComponentMappingMapper componentMappingMapper = ComponentMappingMapper.INSTANCE;
|
||||
DossierTemplatePersistenceService dossierTemplatePersistenceService;
|
||||
|
||||
|
||||
public List<DossierTemplateModel> getAllDossierTemplates() {
|
||||
@ -162,6 +179,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.ruleFileType(ruleFileType)
|
||||
.build();
|
||||
|
||||
DroolsValidationResponse rulesValidationResponse = new DroolsValidationResponse();
|
||||
|
||||
try {
|
||||
@ -194,7 +212,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
|
||||
|
||||
private ResponseEntity<?> downloadRules(String dossierTemplateId, RuleFileType ruleFileType) {
|
||||
|
||||
var ruleEntity = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
|
||||
RuleSetEntity ruleEntity = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
|
||||
|
||||
var data = ruleEntity.getValue().getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
@ -209,4 +227,112 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
|
||||
return new ResponseEntity<>(new InputStreamResource(is), httpHeaders, HttpStatus.OK);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@PreAuthorize("hasAuthority('" + READ_RULES + "')")
|
||||
public ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) {
|
||||
|
||||
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
|
||||
List<com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata> summaries = componentMappingService.getMetaDataByDossierTemplateId(
|
||||
dossierTemplateId);
|
||||
List<ComponentMappingMetadataModel> componentMappingMetadataModelList = componentMappingMapper.toModelList(summaries);
|
||||
return new ComponentMappingSummary(dossierTemplateId, componentMappingMetadataModelList);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
@PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
|
||||
public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, char delimiter) {
|
||||
|
||||
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
|
||||
|
||||
String nameToUse = Strings.isNullOrEmpty(name) ? file.getName().split("\\.")[0] : name;
|
||||
|
||||
if (Strings.isNullOrEmpty(nameToUse)) {
|
||||
throw new BadRequestException("The provided file name is not valid!");
|
||||
}
|
||||
|
||||
Path mappingFile = saveToFile(file);
|
||||
String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename();
|
||||
|
||||
com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
|
||||
nameToUse,
|
||||
fileName,
|
||||
delimiter,
|
||||
encoding,
|
||||
mappingFile.toFile());
|
||||
|
||||
Files.deleteIfExists(mappingFile);
|
||||
|
||||
return componentMappingMapper.toModel(metaData);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
@PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
|
||||
public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String encoding, char delimiter) {
|
||||
|
||||
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
|
||||
|
||||
Path mappingFile = saveToFile(file);
|
||||
|
||||
com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId,
|
||||
componentMappingId,
|
||||
encoding,
|
||||
delimiter,
|
||||
mappingFile.toFile());
|
||||
|
||||
Files.deleteIfExists(mappingFile);
|
||||
|
||||
return componentMappingMapper.toModel(resultMetaData);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@PreAuthorize("hasAuthority('" + READ_RULES + "')")
|
||||
public ResponseEntity<?> downloadMapping(String dossierTemplateId, String componentMappingId) {
|
||||
|
||||
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
|
||||
|
||||
ComponentMappingDownloadModel mappingDownloadModel = componentMappingService.getMappingForDownload(dossierTemplateId, componentMappingId);
|
||||
|
||||
HttpHeaders httpHeaders = new HttpHeaders();
|
||||
httpHeaders.setContentType(MediaType.TEXT_PLAIN);
|
||||
|
||||
httpHeaders.add("Content-Disposition",
|
||||
"attachment"
|
||||
+ "; filename*="
|
||||
+ mappingDownloadModel.encoding().toLowerCase(Locale.US)
|
||||
+ "''"
|
||||
+ StringEncodingUtils.urlEncode(mappingDownloadModel.fileName()));
|
||||
|
||||
return new ResponseEntity<>(mappingDownloadModel.mappingFileResource(), httpHeaders, HttpStatus.OK);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
|
||||
public ResponseEntity<?> deleteMapping(String dossierTemplateId, String componentMappingId) {
|
||||
|
||||
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
|
||||
|
||||
componentMappingService.delete(dossierTemplateId, componentMappingId);
|
||||
|
||||
return new ResponseEntity<>(HttpStatus.NO_CONTENT);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private static Path saveToFile(MultipartFile file) {
|
||||
|
||||
Path mappingFile = Files.createTempFile(file.getName(), ".csv");
|
||||
|
||||
try (var out = new FileOutputStream(mappingFile.toFile())) {
|
||||
out.write(file.getBytes());
|
||||
}
|
||||
return mappingFile;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,184 @@
|
||||
package com.iqser.red.persistence.service.v2.external.api.impl.controller;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.quartz.JobDataMap;
|
||||
import org.quartz.JobKey;
|
||||
import org.quartz.Scheduler;
|
||||
import org.quartz.SchedulerException;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.converter.HttpMessageNotReadableException;
|
||||
import org.springframework.security.access.AccessDeniedException;
|
||||
import org.springframework.web.bind.MethodArgumentNotValidException;
|
||||
import org.springframework.web.bind.WebDataBinder;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.InitBinder;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
import org.springframework.web.multipart.support.MissingServletRequestPartException;
|
||||
|
||||
import com.fasterxml.jackson.databind.exc.InvalidFormatException;
|
||||
import com.iqser.red.commons.spring.ErrorMessage;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
import com.mchange.rmi.NotAuthorizedException;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RestControllerAdvice
|
||||
@RequiredArgsConstructor
|
||||
@Order(Ordered.HIGHEST_PRECEDENCE)
|
||||
public class ExternalControllerAdviceV2 {
|
||||
|
||||
private final Scheduler scheduler;
|
||||
|
||||
|
||||
@Hidden
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.NOT_FOUND)
|
||||
@ExceptionHandler(value = NotFoundException.class)
|
||||
public ErrorMessage handleContentNotFoundException(NotFoundException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
/* error handling */
|
||||
|
||||
|
||||
@Hidden
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.BAD_REQUEST)
|
||||
@ExceptionHandler(value = BadRequestException.class)
|
||||
public ErrorMessage handleBadRequestException(BadRequestException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@Hidden
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.CONFLICT)
|
||||
@ExceptionHandler(value = {ConflictException.class})
|
||||
protected ErrorMessage handleConflictException(ConflictException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.FORBIDDEN)
|
||||
@ExceptionHandler({AccessDeniedException.class})
|
||||
public ErrorMessage handleAccessDeniedException(AccessDeniedException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.UNAUTHORIZED)
|
||||
@ExceptionHandler({NotAuthorizedException.class})
|
||||
public ErrorMessage handleNotAuthorizedException(NotAuthorizedException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.FORBIDDEN)
|
||||
@ExceptionHandler({NotAllowedException.class})
|
||||
public ErrorMessage handleNotAllowedException(NotAllowedException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@Hidden
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
|
||||
@ExceptionHandler({org.springframework.security.acls.model.NotFoundException.class})
|
||||
public ErrorMessage handleACLNotFound(org.springframework.security.acls.model.NotFoundException e) {
|
||||
|
||||
// in case this error occurs on a rest request / force trigger the sync job
|
||||
try {
|
||||
scheduler.triggerJob(new JobKey("SyncUserPermissionsJob"), new JobDataMap(Map.of("tenantId", TenantContext.getTenantId())));
|
||||
} catch (SchedulerException ex) {
|
||||
log.debug("Failed to force trigger SyncUserPermissionsJob", ex);
|
||||
}
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@Hidden
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.BAD_REQUEST)
|
||||
@ExceptionHandler({MethodArgumentNotValidException.class})
|
||||
public ErrorMessage handleMethodArgumentNotValidException(MethodArgumentNotValidException e) {
|
||||
|
||||
var errorList = e.getFieldErrors();
|
||||
String errorListAsString = errorList.stream()
|
||||
.map(fieldError -> fieldError.getField() + ": " + fieldError.getDefaultMessage())
|
||||
.collect(Collectors.joining(", "));
|
||||
return new ErrorMessage(OffsetDateTime.now(), String.format("You have empty/wrong formatted parameters: %s", errorListAsString));
|
||||
}
|
||||
|
||||
|
||||
@Hidden
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.BAD_REQUEST)
|
||||
@ExceptionHandler({MissingServletRequestPartException.class})
|
||||
public ErrorMessage handleMissingServletRequestPartException(MissingServletRequestPartException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@Hidden
|
||||
@ResponseBody
|
||||
@ResponseStatus(HttpStatus.BAD_REQUEST)
|
||||
@ExceptionHandler({HttpMessageNotReadableException.class})
|
||||
public ErrorMessage handleHttpMessageNotReadableException(HttpMessageNotReadableException e) {
|
||||
|
||||
var cause = e.getCause();
|
||||
if (cause instanceof InvalidFormatException) {
|
||||
InvalidFormatException invalidFormatException = (InvalidFormatException) cause;
|
||||
|
||||
Class<?> targetType = invalidFormatException.getTargetType();
|
||||
if (targetType != null && targetType.isEnum()) {
|
||||
return new ErrorMessage(OffsetDateTime.now(), String.format("Unsupported value for %s", targetType.getSimpleName()));
|
||||
}
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), cause.getMessage());
|
||||
}
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@Order(10000)
|
||||
public static class BinderControllerAdvice {
|
||||
|
||||
@InitBinder
|
||||
public void setAllowedFields(WebDataBinder dataBinder) {
|
||||
// This code protects Spring Core from a "Remote Code Execution" attack (dubbed "Spring4Shell").
|
||||
// By applying this mitigation, you prevent the "Class Loader Manipulation" attack vector from firing.
|
||||
// For more details, see this post: https://www.lunasec.io/docs/blog/spring-rce-vulnerabilities/
|
||||
String[] denylist = new String[]{"class.*", "Class.*", "*.class.*", "*.Class.*"};
|
||||
dataBinder.setDisallowedFields(denylist);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package com.iqser.red.persistence.service.v2.external.api.impl.mapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel;
|
||||
|
||||
@Mapper
|
||||
public interface ComponentMappingMapper {
|
||||
|
||||
ComponentMappingMapper INSTANCE = Mappers.getMapper(ComponentMappingMapper.class);
|
||||
|
||||
|
||||
ComponentMappingMetadataModel toModel(ComponentMappingMetadata componentMappingMetadata);
|
||||
|
||||
|
||||
List<ComponentMappingMetadataModel> toModelList(List<ComponentMappingMetadata> componentMappingMetadata);
|
||||
|
||||
|
||||
ComponentMappingMetadata toDto(ComponentMappingMetadataModel componentMappingMetadataModel);
|
||||
|
||||
|
||||
List<ComponentMappingMetadata> toDtoList(List<ComponentMappingMetadataModel> componentMappingMetadataModels);
|
||||
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package com.iqser.red.service.persistence.service.v2.api.external.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class ComponentMappingMetadataModel {
|
||||
|
||||
String id;
|
||||
String name;
|
||||
String fileName;
|
||||
Integer version;
|
||||
List<String> columnLabels;
|
||||
Integer numberOfLines;
|
||||
String encoding;
|
||||
char delimiter;
|
||||
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
package com.iqser.red.service.persistence.service.v2.api.external.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class ComponentMappingSummary {
|
||||
|
||||
String dossierTemplateId;
|
||||
List<ComponentMappingMetadataModel> componentMappingList;
|
||||
|
||||
}
|
||||
@ -8,20 +8,29 @@ import static com.iqser.red.service.persistence.service.v2.api.external.resource
|
||||
import static com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource.DOSSIER_TEMPLATE_PATH;
|
||||
import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_ID_PARAM;
|
||||
import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_ID_PATH_VARIABLE;
|
||||
import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_PATH;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RequestPart;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponentsList;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||
|
||||
@ -29,7 +38,8 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
public interface ComponentResource {
|
||||
|
||||
String PATH = ExternalApiConstants.BASE_PATH + DOSSIER_TEMPLATE_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + DOSSIER_PATH + DOSSIER_ID_PATH_PARAM + FILE_PATH;
|
||||
String PATH = ExternalApiConstants.BASE_PATH + DOSSIER_TEMPLATE_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE;
|
||||
String FILE_PATH = PATH + DOSSIER_PATH + DOSSIER_ID_PATH_PARAM + FileResource.FILE_PATH;
|
||||
|
||||
String COMPONENTS_PATH = "/components";
|
||||
|
||||
@ -39,13 +49,13 @@ public interface ComponentResource {
|
||||
|
||||
String INCLUDE_DETAILS_DESCRIPTION = """
|
||||
A toggle to decide whether to include detailed component information in the response:
|
||||
|
||||
- true: The component object's field componentDetails stores detailed information about the source of its respective value(s).
|
||||
- false (default): The component object does not contain a field componentDetails.
|
||||
""";
|
||||
|
||||
|
||||
@GetMapping(value = PATH + FILE_ID_PATH_VARIABLE + COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE})
|
||||
|
||||
@GetMapping(value = FILE_PATH + FILE_ID_PATH_VARIABLE + COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE})
|
||||
@Operation(summary = "Returns the components for a file", description = "None")
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
|
||||
FileComponents getComponents(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
|
||||
@ -54,11 +64,13 @@ public interface ComponentResource {
|
||||
@Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails);
|
||||
|
||||
|
||||
@GetMapping(value = PATH + BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE})
|
||||
@GetMapping(value = FILE_PATH + BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE})
|
||||
@Operation(summary = "Returns the components for all files of a dossier", description = "None")
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
|
||||
FileComponentsList getComponentsOfDossier(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
|
||||
@Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier that contains the file.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
|
||||
@Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails);
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.service.v2.api.external.resource;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary;
|
||||
import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
@ -13,9 +15,11 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RequestPart;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
@ -33,15 +37,20 @@ public interface DossierTemplateResource {
|
||||
|
||||
String ENTITY_RULES_PATH = "/entity-rules";
|
||||
String COMPONENT_RULES_PATH = "/component-rules";
|
||||
String COMPONENT_MAPPINGS_PATH = "/component-mappings";
|
||||
|
||||
String FILE_ATTRIBUTE_DEFINITIONS_PATH = "/file-attribute-definitions";
|
||||
String DOSSIER_TEMPLATE_ID_PARAM = "dossierTemplateId";
|
||||
String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID_PARAM + "}";
|
||||
|
||||
String RULE_FILE_TYPE_PARAMETER_NAME = "ruleFileType";
|
||||
|
||||
String COMPONENT_MAPPING_ID_PARAM = "componentMappingId";
|
||||
String COMPONENT_MAPPING_ID_PATH_VARIABLE = "/{" + COMPONENT_MAPPING_ID_PARAM + "}";
|
||||
|
||||
String DRY_RUN_PARAM = "dryRun";
|
||||
|
||||
String ENCODING_PARAM = "encoding";
|
||||
String DELIMITER_PARAM = "delimiter";
|
||||
String MAPPING_NAME_PARAM = "name";
|
||||
|
||||
@GetMapping(value = PATH, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(summary = "Lists all existing DossierTemplates.", description = "None")
|
||||
@ -60,8 +69,8 @@ public interface DossierTemplateResource {
|
||||
@Operation(summary = "Upload a component or entity rules file in drools format for a specific DossierTemplate.")
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Rules upload successful."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found."), @ApiResponse(responseCode = "400", description = "Uploaded rules could not be verified."), @ApiResponse(responseCode = "422", description = "Uploaded rules could not be compiled.")})
|
||||
ResponseEntity<DroolsValidationResponse> uploadEntityRules(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
|
||||
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
|
||||
@Parameter(name = DRY_RUN_PARAM, description = "If true rules will be only validated not stored.") @RequestParam(value = DRY_RUN_PARAM, required = false, defaultValue = "false") boolean dryRun);
|
||||
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
|
||||
@Parameter(name = DRY_RUN_PARAM, description = "If true rules will be only validated not stored.") @RequestParam(value = DRY_RUN_PARAM, required = false, defaultValue = "false") boolean dryRun);
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ -96,4 +105,51 @@ public interface DossierTemplateResource {
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "File attribute definitions returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")})
|
||||
FileAttributeDefinitionList getFileAttributeDefinitions(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId);
|
||||
|
||||
|
||||
@Operation(summary = "Get the component mapping summaries of a DossierTemplate.", description = "None")
|
||||
@GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping views returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")})
|
||||
ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId);
|
||||
|
||||
|
||||
@Operation(summary = "Upload a new component mapping to a DossierTemplate.", description = "None")
|
||||
@PostMapping(value = PATH
|
||||
+ DOSSIER_TEMPLATE_ID_PATH_VARIABLE
|
||||
+ COMPONENT_MAPPINGS_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping uploaded successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")})
|
||||
ComponentMappingMetadataModel uploadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
|
||||
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
|
||||
@Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
|
||||
@Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
|
||||
@Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter);
|
||||
|
||||
|
||||
@Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None")
|
||||
@PutMapping(value = PATH
|
||||
+ DOSSIER_TEMPLATE_ID_PATH_VARIABLE
|
||||
+ COMPONENT_MAPPINGS_PATH
|
||||
+ COMPONENT_MAPPING_ID_PATH_VARIABLE, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping updated successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")})
|
||||
ComponentMappingMetadataModel updateMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
|
||||
@PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId,
|
||||
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
|
||||
@Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
|
||||
@Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter);
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
@Operation(summary = "Returns file containing the specified mapping as a file.")
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")})
|
||||
@GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE)
|
||||
ResponseEntity<?> downloadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId);
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
@Operation(summary = "Deletes a specified mapping.")
|
||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")})
|
||||
@DeleteMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE)
|
||||
ResponseEntity<?> deleteMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId);
|
||||
|
||||
}
|
||||
|
||||
@ -276,7 +276,7 @@ paths:
|
||||
tags:
|
||||
- 1. Dossier Templates
|
||||
description: |
|
||||
Retrieves a collection of file attribute definitions associated with a specific dossier template. Each file
|
||||
Use this endpoint to retrieves a collection of file attribute definitions associated with a specific dossier template. Each file
|
||||
attribute definition includes details such as attribute type, name, and other relevant metadata. This endpoint
|
||||
is useful for clients needing to understand what attributes are expected or allowed for files under a particular
|
||||
dossier template.
|
||||
@ -290,6 +290,214 @@ paths:
|
||||
$ref: '#/components/schemas/FileAttributeDefinitionList'
|
||||
description: |
|
||||
Successfully returned the file attribute definitions for the specified dossier template.
|
||||
/api/dossier-templates/{dossierTemplateId}/component-mappings:
|
||||
get:
|
||||
operationId: listAllMappings
|
||||
tags:
|
||||
- 1. Dossier Templates
|
||||
summary: Returns a list of all existing component mappings in a dossier template
|
||||
description: |
|
||||
Use this endpoint to retrieve a summary of all component mappings associated with a specific DossierTemplate.
|
||||
|
||||
The summary consists of the stored metadata of a component mapping file.
|
||||
This endpoint is useful for clients to understand the available mappings under a particular DossierTemplate.
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/dossierTemplateId'
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ComponentMappingList'
|
||||
description: |
|
||||
Successfully returned the component mapping summary for the specified dossier template.
|
||||
"400":
|
||||
$ref: '#/components/responses/400'
|
||||
"401":
|
||||
$ref: '#/components/responses/401'
|
||||
"403":
|
||||
$ref: '#/components/responses/403'
|
||||
"404":
|
||||
$ref: '#/components/responses/404-mapping'
|
||||
"429":
|
||||
$ref: '#/components/responses/429'
|
||||
"500":
|
||||
$ref: '#/components/responses/500'
|
||||
post:
|
||||
operationId: uploadMapping
|
||||
summary: Upload a new component mapping to a DossierTemplate.
|
||||
description: |
|
||||
Use this endpoint to upload a new component mapping to a specific DossierTemplate.
|
||||
|
||||
#### File Requirements
|
||||
- **Format:** The file must be in CSV (comma-separated values) format.
|
||||
- **Header Row:** The first row should contain the column labels.
|
||||
- **Data Consistency:** All rows must have the same number of columns to ensure rectangular data structure.
|
||||
|
||||
#### Sorting and Performance
|
||||
- **Sorting:** Rows are automatically sorted by the values in each column, from left to right, to enhance lookup speed.
|
||||
- **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
|
||||
|
||||
#### Customization Options
|
||||
- Users can specify the delimiter and encoding used in the CSV file.
|
||||
|
||||
#### Usage
|
||||
- The component mapping file can be utilized in component rules to relate components to existing master data.
|
||||
|
||||
#### Example
|
||||
|
||||
| search_value | mapped_value |
|
||||
|--------------|--------------|
|
||||
| Alice | Manager |
|
||||
| Bob | Developer |
|
||||
| Charlie | Analyst |
|
||||
tags:
|
||||
- 1. Dossier Templates
|
||||
requestBody:
|
||||
content:
|
||||
multipart/form-data:
|
||||
schema:
|
||||
$ref: '#/components/schemas/UploadRequest'
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/dossierTemplateId'
|
||||
- $ref: '#/components/parameters/mappingName'
|
||||
- $ref: '#/components/parameters/encoding'
|
||||
- $ref: '#/components/parameters/delimiter'
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ComponentMapping'
|
||||
description: |
|
||||
Component mapping uploaded successfully and returned the component mapping metadata.
|
||||
"400":
|
||||
$ref: '#/components/responses/400'
|
||||
"401":
|
||||
$ref: '#/components/responses/401'
|
||||
"403":
|
||||
$ref: '#/components/responses/403'
|
||||
"404":
|
||||
$ref: '#/components/responses/404-mapping'
|
||||
"429":
|
||||
$ref: '#/components/responses/429'
|
||||
"500":
|
||||
$ref: '#/components/responses/500'
|
||||
|
||||
/api/dossier-templates/{dossierTemplateId}/component-mappings/{comonentMappingId}:
|
||||
get:
|
||||
operationId: downloadMappingFile
|
||||
tags:
|
||||
- 1. Dossier Templates
|
||||
summary: Download a specific component mapping file of a specific dossier template.
|
||||
description: |
|
||||
Use this endpoint to download a specific component mapping file of a designated DossierTemplate.
|
||||
|
||||
- The file retains its original name and encoding as when it was uploaded.
|
||||
- The sorting of the file may have changed to enable faster lookups.
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/dossierTemplateId'
|
||||
- $ref: '#/components/parameters/componentMappingId'
|
||||
responses:
|
||||
"200":
|
||||
headers:
|
||||
Content-Disposition:
|
||||
schema:
|
||||
type: string
|
||||
example: attachment; filename*=mapping.csv
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
description: |
|
||||
Successfully downloaded the requested component mapping.
|
||||
"400":
|
||||
$ref: '#/components/responses/400'
|
||||
"401":
|
||||
$ref: '#/components/responses/401'
|
||||
"403":
|
||||
$ref: '#/components/responses/403'
|
||||
"404":
|
||||
$ref: '#/components/responses/404-mapping'
|
||||
"429":
|
||||
$ref: '#/components/responses/429'
|
||||
"500":
|
||||
$ref: '#/components/responses/500'
|
||||
put:
|
||||
operationId: updateMapping
|
||||
summary: Update an existing component mapping of a DossierTemplate.
|
||||
description: |
|
||||
Use this endpoint to update an existing component mapping of a specific dossier template.
|
||||
|
||||
#### File Requirements
|
||||
- **Format:** The file must be in CSV (comma-separated values) format.
|
||||
- **Header Row:** The first row should contain the column labels.
|
||||
- **Data Consistency:** All rows must have the same number of columns to ensure rectangular data structure.
|
||||
|
||||
#### Sorting and Performance
|
||||
- **Sorting:** Rows are automatically sorted by the values in each column, from left to right, to enhance lookup speed.
|
||||
- **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
|
||||
|
||||
#### Customization Options
|
||||
- Users can specify the delimiter and encoding used in the CSV file.
|
||||
tags:
|
||||
- 1. Dossier Templates
|
||||
requestBody:
|
||||
content:
|
||||
multipart/form-data:
|
||||
schema:
|
||||
$ref: '#/components/schemas/UploadRequest'
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/dossierTemplateId'
|
||||
- $ref: '#/components/parameters/componentMappingId'
|
||||
- $ref: '#/components/parameters/encoding'
|
||||
- $ref: '#/components/parameters/delimiter'
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ComponentMapping'
|
||||
description: |
|
||||
Component mapping uploaded successfully and returned the component mapping metadata.
|
||||
"400":
|
||||
$ref: '#/components/responses/400'
|
||||
"401":
|
||||
$ref: '#/components/responses/401'
|
||||
"403":
|
||||
$ref: '#/components/responses/403'
|
||||
"404":
|
||||
$ref: '#/components/responses/404-mapping'
|
||||
"429":
|
||||
$ref: '#/components/responses/429'
|
||||
"500":
|
||||
$ref: '#/components/responses/500'
|
||||
delete:
|
||||
operationId: deleteMappingFile
|
||||
tags:
|
||||
- 1. Dossier Templates
|
||||
summary: Delete a specific component mapping file of a specific dossier template.
|
||||
description: |
|
||||
Use this endpoint to delete a specific component mapping file of a designated dossier template.
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/dossierTemplateId'
|
||||
- $ref: '#/components/parameters/componentMappingId'
|
||||
responses:
|
||||
"204":
|
||||
description: |
|
||||
Successfully deleted the requested component mapping.
|
||||
"400":
|
||||
$ref: '#/components/responses/400'
|
||||
"401":
|
||||
$ref: '#/components/responses/401'
|
||||
"403":
|
||||
$ref: '#/components/responses/403'
|
||||
"404":
|
||||
$ref: '#/components/responses/404-mapping'
|
||||
"429":
|
||||
$ref: '#/components/responses/429'
|
||||
"500":
|
||||
$ref: '#/components/responses/500'
|
||||
/api/dossier-templates/{dossierTemplateId}/dossiers:
|
||||
get:
|
||||
operationId: getDossiers
|
||||
@ -304,7 +512,7 @@ paths:
|
||||
|
||||
Use this endpoint to fetch the required dossiers before performing actions on specific ones. Use the query parameters
|
||||
to modify the response. E.g., set the `includeArchivedDossiers` parameter to `true` so that the response also contains
|
||||
*archived* dossiers.
|
||||
*archived* dossiers.
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/dossierTemplateId'
|
||||
- $ref: '#/components/parameters/includeActiveDossiers'
|
||||
@ -792,14 +1000,14 @@ components:
|
||||
responses:
|
||||
"400":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
The request was malformed or invalid. Double-check the request structure or parameters.
|
||||
"401":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
@ -807,21 +1015,28 @@ components:
|
||||
resource. This error can happen if the access token was revoked or has expired.
|
||||
"403":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
Forbidden. Your credentials are valid, but you do not have permission to access this resource.
|
||||
"404-dossier-template":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
Dossier template not found. This happens if the requested dossier template does not exist.
|
||||
"404-mapping":
|
||||
content:
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
Dossier template or component mapping not found. This happens if the requested dossier template or component mapping does not exist.
|
||||
"404-dossier":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
@ -831,7 +1046,7 @@ components:
|
||||
for a previously existing dossier only if it is actually deleted permanently.
|
||||
"404-file":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
@ -841,28 +1056,28 @@ components:
|
||||
only if the file is deleted permanently.
|
||||
"409-dossier-conflict":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
Name conflict: The provided name is already in use by another dossier. It needs to be unique in the scope of your workspace.
|
||||
422-rules:
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/RuleValidation'
|
||||
description: |
|
||||
Invalid rules file: There were validation errors, the rules file is unprocessable.
|
||||
"429":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: |
|
||||
Too many requests have been made in a given time frame. Rate limiting is in place to prevent abuse.
|
||||
"500":
|
||||
content:
|
||||
'*/*':
|
||||
'application/json':
|
||||
schema:
|
||||
$ref: '#/components/schemas/ErrorMessage'
|
||||
description: Internal Server Error. An unexpected error occurred on the server side. Please try again later or contact support.
|
||||
@ -876,6 +1091,15 @@ components:
|
||||
style: simple
|
||||
explode: false
|
||||
description: The identifier of a dossier template
|
||||
componentMappingId:
|
||||
name: componentMappingId
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
style: simple
|
||||
explode: false
|
||||
description: The identifier of a component mapping
|
||||
dryRun:
|
||||
name: dryRun
|
||||
in: query
|
||||
@ -888,6 +1112,41 @@ components:
|
||||
description: |
|
||||
A toggle to activate the dry-run mode: If set to `false` (default), the request will update the system.
|
||||
If set to `true`, the request will just be evaluated without actual changes in the system.
|
||||
encoding:
|
||||
name: encoding
|
||||
required: false
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
- UTF-8
|
||||
- UTF-16
|
||||
- UTF_16BE
|
||||
- UTF_16LE
|
||||
- ISO-8859-1
|
||||
- US-ASCII
|
||||
example: UTF-8
|
||||
default: UTF-8
|
||||
description: "An identifier for the used encoding of the file. Only java's standard charsets are supported."
|
||||
delimiter:
|
||||
name: delimiter
|
||||
required: false
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
minLength: 1
|
||||
maxLength: 1
|
||||
example: ','
|
||||
default: ','
|
||||
description: "The delimiter used as a separator in a csv file."
|
||||
mappingName:
|
||||
name: name
|
||||
required: false
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
example: "MasterDataMapping"
|
||||
description: "The name with which the mapping should be associated with. If none is provided, the file name will be used."
|
||||
dossierId:
|
||||
name: dossierId
|
||||
in: path
|
||||
@ -1848,6 +2107,108 @@ components:
|
||||
- daadea5f-917b-482a-b7d2-e65afe8f80ca
|
||||
- 8130acf6-4910-4123-827c-caacd8111402
|
||||
dossierStatusId: b8280985-f558-43c0-852a-a3fa86803548
|
||||
ComponentMapping:
|
||||
description: |
|
||||
The `ComponentMapping` object represents the metadata of a component mapping csv file. These CSV files may be used in the component rules to relate components to an existing knowledge base.
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
description: |
|
||||
A unique identifier for the component mapping metadata. It's generated automatically. Use this to retrieve a specific component mapping file.
|
||||
type: string
|
||||
name:
|
||||
description: |
|
||||
The name of the component mapping.
|
||||
type: string
|
||||
fileName:
|
||||
description: |
|
||||
The name of the uploaded component mapping file.
|
||||
type: string
|
||||
version:
|
||||
description: |
|
||||
The version of the file. It is incremented with each update to the file.
|
||||
type: integer
|
||||
columnLabels:
|
||||
description: |
|
||||
A list of column labels found in the component mapping file.
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
numberOfLines:
|
||||
description: |
|
||||
The number of lines in the component mapping file.
|
||||
type: integer
|
||||
format: int32
|
||||
encoding:
|
||||
description: |
|
||||
The encoding used for the component mapping file.
|
||||
type: string
|
||||
delimiter:
|
||||
description: |
|
||||
The delimiter used for separating values in the component mapping file.
|
||||
type: string
|
||||
required:
|
||||
- id
|
||||
- name
|
||||
- fileName
|
||||
- columnLabels
|
||||
- numberOfLines
|
||||
- encoding
|
||||
- delimiter
|
||||
example:
|
||||
id: 24ff9c3c-4863-4aea-8eda-cab8838b9192
|
||||
name: MasterDataMapping
|
||||
fileName: master_data.csv
|
||||
columnLabels:
|
||||
- Column1
|
||||
- Column2
|
||||
- Column3
|
||||
numberOfLines: 100
|
||||
encoding: UTF-8
|
||||
delimiter: ','
|
||||
ComponentMappingList:
|
||||
description: |
|
||||
The `ComponentMappingList` object represents a collection of ComponentMapping.
|
||||
type: object
|
||||
example:
|
||||
dossierTemplateId: 1e07cde0-d36a-4ab7-b389-494ca694a0cb
|
||||
componentMappingList:
|
||||
- id: 24ff9c3c-4863-4aea-8eda-cab8838b9192
|
||||
name: MasterDataMapping
|
||||
fileName: master_data.csv
|
||||
columnLabels:
|
||||
- Column1
|
||||
- Column2
|
||||
- Column3
|
||||
numberOfLines: 100
|
||||
encoding: UTF-8
|
||||
delimiter: ','
|
||||
- id: 2e07cde0-d36a-4ab7-b389-494ca694a0cb
|
||||
name: RegulationNameMapping
|
||||
fileName: regulation-names.csv
|
||||
columnLabels:
|
||||
- Year
|
||||
- Title
|
||||
- Identifier
|
||||
- Name
|
||||
numberOfLines: 150
|
||||
encoding: UTF-16
|
||||
delimiter: ;
|
||||
properties:
|
||||
dossierTemplateId:
|
||||
description: |
|
||||
The identifier of the dossier template associated with the ComponentMappingList.
|
||||
type: string
|
||||
format: uuid
|
||||
componentMappingList:
|
||||
description: |
|
||||
A list of component mapping metadata associated with this dossier template.
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/ComponentMapping"
|
||||
required:
|
||||
- dossierTemplateId
|
||||
- componentMappingList
|
||||
DossierTemplate:
|
||||
description: |
|
||||
The `DossierTemplate` object represents the blueprint for creating and
|
||||
|
||||
@ -53,6 +53,7 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP
|
||||
|
||||
public void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AnalyzeResult analyzeResult) {
|
||||
|
||||
log.info("Received analysis result {}", analyzeResult);
|
||||
fileStatusProcessingUpdateService.analysisSuccessful(dossierId, fileId, analyzeResult);
|
||||
}
|
||||
|
||||
|
||||
@ -61,6 +61,7 @@ dependencies {
|
||||
api("com.opencsv:opencsv:5.4")
|
||||
api("org.springframework.cloud:spring-cloud-starter-openfeign:${springCloudVersion}")
|
||||
api("commons-validator:commons-validator:1.7")
|
||||
api("com.opencsv:opencsv:5.9")
|
||||
|
||||
implementation("org.mapstruct:mapstruct:1.5.5.Final")
|
||||
annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final")
|
||||
|
||||
@ -0,0 +1,78 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.entity;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.hibernate.annotations.Fetch;
|
||||
import org.springframework.data.annotation.LastModifiedDate;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
|
||||
|
||||
import jakarta.persistence.CollectionTable;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.ElementCollection;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
import jakarta.persistence.Table;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.NonNull;
|
||||
|
||||
@Data
|
||||
@Entity
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Table(name = "component_mappings")
|
||||
public class ComponentMappingEntity {
|
||||
|
||||
@Id
|
||||
@Column(name = "id")
|
||||
String id;
|
||||
|
||||
@NonNull
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
private DossierTemplateEntity dossierTemplate;
|
||||
|
||||
@NonNull
|
||||
@Builder.Default
|
||||
String storageId = "";
|
||||
|
||||
@NonNull
|
||||
@Builder.Default
|
||||
String name = "";
|
||||
|
||||
@NonNull
|
||||
@Builder.Default
|
||||
String fileName = "";
|
||||
|
||||
@NonNull
|
||||
@Builder.Default
|
||||
Integer version = -1;
|
||||
|
||||
@LastModifiedDate
|
||||
OffsetDateTime changedDate;
|
||||
|
||||
@NonNull
|
||||
@Builder.Default
|
||||
@ElementCollection(fetch = FetchType.EAGER)
|
||||
@CollectionTable(name = "component_mapping_column_labels")
|
||||
List<String> columnLabels = new ArrayList<>();
|
||||
|
||||
@NonNull
|
||||
@Builder.Default
|
||||
Integer numberOfLines = 0;
|
||||
|
||||
@NonNull
|
||||
@Builder.Default
|
||||
String encoding = "UTF-8";
|
||||
|
||||
@Builder.Default
|
||||
char delimiter = ',';
|
||||
|
||||
}
|
||||
@ -69,8 +69,8 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation {
|
||||
@Column
|
||||
private OffsetDateTime softDeletedTime;
|
||||
|
||||
@ElementCollection(fetch = FetchType.EAGER)
|
||||
@Fetch(FetchMode.SUBSELECT)
|
||||
@ElementCollection(fetch = FetchType.EAGER)
|
||||
private List<RectangleEntity> positions = new ArrayList<>();
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
|
||||
@ -14,8 +14,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
|
||||
import jakarta.persistence.CascadeType;
|
||||
import jakarta.persistence.CollectionTable;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Convert;
|
||||
import jakarta.persistence.ElementCollection;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EnumType;
|
||||
import jakarta.persistence.Enumerated;
|
||||
@ -197,6 +199,9 @@ public class FileEntity {
|
||||
@Column
|
||||
private OffsetDateTime errorTimestamp;
|
||||
|
||||
@ElementCollection
|
||||
private List<FileEntityComponentMappingVersionEntity> componentMappingVersions;
|
||||
|
||||
|
||||
public OffsetDateTime getLastOCRTime() {
|
||||
|
||||
|
||||
@ -0,0 +1,17 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
|
||||
|
||||
import jakarta.persistence.Embeddable;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@Embeddable
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class FileEntityComponentMappingVersionEntity {
|
||||
|
||||
private String name;
|
||||
private Integer version;
|
||||
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.mapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
|
||||
@Mapper
|
||||
public interface ComponentMappingEntityMapper {
|
||||
|
||||
ComponentMappingEntityMapper INSTANCE = Mappers.getMapper(ComponentMappingEntityMapper.class);
|
||||
|
||||
|
||||
ComponentMappingMetadata toComponentMappingMetaData(ComponentMappingEntity componentMappingMetaData);
|
||||
|
||||
|
||||
List<ComponentMappingMetadata> toComponentMappingMetaDataList(List<ComponentMappingEntity> componentMappingEntities);
|
||||
|
||||
|
||||
ComponentMappingEntity toComponentMappingEntity(ComponentMappingMetadata componentMappingSummary);
|
||||
|
||||
|
||||
List<ComponentMappingEntity> toComponentMappingEntityList(List<ComponentMappingMetadata> componentMappingSummary);
|
||||
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.model;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
|
||||
public record ComponentMapping(ComponentMappingMetadata metaData, File file) {
|
||||
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.model;
|
||||
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
|
||||
public record ComponentMappingDownloadModel(InputStreamResource mappingFileResource, String encoding, String fileName) {
|
||||
|
||||
}
|
||||
@ -14,6 +14,5 @@ public class DownloadJob {
|
||||
|
||||
private String userId;
|
||||
private String storageId;
|
||||
private Boolean includeUnprocessed;
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,104 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ComponentMappingRepository;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Service
|
||||
@Transactional
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class ComponentMappingPersistenceService {
|
||||
|
||||
StorageService storageService;
|
||||
|
||||
ComponentMappingRepository repository;
|
||||
|
||||
|
||||
public List<ComponentMappingEntity> getByDossierTemplateId(String dossierTemplateId) {
|
||||
|
||||
return repository.findByDossierTemplateId(dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
public void deleteById(String dossierTemplateId, String id) {
|
||||
|
||||
ComponentMappingEntity entity = getEntityById(dossierTemplateId, id);
|
||||
delete(entity);
|
||||
}
|
||||
|
||||
|
||||
public ComponentMappingEntity getEntityById(String dossierTemplateId, String id) {
|
||||
|
||||
return repository.findByIdAndDossierTemplateId(id, dossierTemplateId)
|
||||
.orElseThrow(() -> new NotFoundException("ComponentMapping with id " + id + " not found!"));
|
||||
}
|
||||
|
||||
|
||||
private void delete(ComponentMappingEntity entity) {
|
||||
|
||||
repository.delete(entity);
|
||||
storageService.deleteObject(TenantContext.getTenantId(), entity.getStorageId());
|
||||
}
|
||||
|
||||
|
||||
public void deleteByDossierTemplateId(String dossierTemplateId) {
|
||||
|
||||
repository.findByDossierTemplateId(dossierTemplateId)
|
||||
.forEach(this::delete);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void updateOrCreate(String storageId, File mappingFile, ComponentMappingEntity entity) {
|
||||
|
||||
entity.setChangedDate(OffsetDateTime.now());
|
||||
repository.saveAndFlush(entity);
|
||||
try (var in = new FileInputStream(mappingFile)) {
|
||||
storageService.storeObject(TenantContext.getTenantId(), storageId, in);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public boolean existsByNameAndDossierTemplateId(String name, String dossierTemplateId) {
|
||||
|
||||
return repository.existsByNameAndDossierTemplateId(name, dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public ComponentMappingDownloadModel getMappingFileForDownload(String dossierTemplateId, String componentMappingId) {
|
||||
|
||||
var entity = getEntityById(dossierTemplateId, componentMappingId);
|
||||
if (!storageService.objectExists(TenantContext.getTenantId(), entity.getStorageId())) {
|
||||
throw new NotFoundException("ComponentMapping with id " + componentMappingId + " does not exist!");
|
||||
}
|
||||
return new ComponentMappingDownloadModel(storageService.getObject(TenantContext.getTenantId(), entity.getStorageId()), entity.getEncoding(), entity.getFileName());
|
||||
}
|
||||
|
||||
|
||||
public File downloadMappingFileToFolder(String storageId, String fileName, Path outputDir) {
|
||||
|
||||
File outputFile = outputDir.resolve(fileName).toFile();
|
||||
storageService.downloadTo(TenantContext.getTenantId(), storageId, outputFile);
|
||||
return outputFile;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,225 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.IllegalCharsetNameException;
|
||||
import java.nio.charset.UnsupportedCharsetException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.mapper.ComponentMappingEntityMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
import com.opencsv.CSVParserBuilder;
|
||||
import com.opencsv.CSVReader;
|
||||
import com.opencsv.CSVReaderBuilder;
|
||||
import com.opencsv.CSVWriter;
|
||||
import com.opencsv.exceptions.CsvException;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class ComponentMappingService {
|
||||
|
||||
ComponentMappingPersistenceService componentMappingPersistenceService;
|
||||
DossierTemplatePersistenceService dossierTemplatePersistenceService;
|
||||
ComponentMappingEntityMapper mappingEntityMapper = ComponentMappingEntityMapper.INSTANCE;
|
||||
|
||||
static CSVSorter CSV_SORTER = new CSVSorter();
|
||||
|
||||
|
||||
public List<ComponentMappingMetadata> getMetaDataByDossierTemplateId(String dossierTemplateId) {
|
||||
|
||||
List<ComponentMappingEntity> entities = componentMappingPersistenceService.getByDossierTemplateId(dossierTemplateId);
|
||||
return mappingEntityMapper.toComponentMappingMetaDataList(entities);
|
||||
}
|
||||
|
||||
|
||||
public ComponentMappingMetadata getMetaData(String dossierTemplateId, String mappingId) {
|
||||
|
||||
return mappingEntityMapper.toComponentMappingMetaData(componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId));
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public ComponentMappingMetadata update(String dossierTemplateId, String mappingId, String encoding, char delimiter, File mappingFile) {
|
||||
|
||||
ComponentMappingEntity entity = componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId);
|
||||
|
||||
return updateOrCreate(entity, encoding, delimiter, mappingFile);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile) {
|
||||
|
||||
if (componentMappingPersistenceService.existsByNameAndDossierTemplateId(name, dossierTemplateId)) {
|
||||
throw new BadRequestException("A mapping with this name already exists in the dossier template!");
|
||||
}
|
||||
|
||||
String id = UUID.randomUUID().toString();
|
||||
String storageId = buildStorageId(dossierTemplateId, id, name, fileName);
|
||||
ComponentMappingEntity entity = ComponentMappingEntity.builder()
|
||||
.id(id)
|
||||
.dossierTemplate(dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId))
|
||||
.storageId(storageId)
|
||||
.name(name)
|
||||
.fileName(fileName)
|
||||
.build();
|
||||
|
||||
return updateOrCreate(entity, encoding, delimiter, mappingFile);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity, String encoding, char delimiter, File mappingFile) {
|
||||
|
||||
Charset charset = resolveCharset(encoding);
|
||||
|
||||
CsvStats stats = sortCSVFile(delimiter, mappingFile, charset);
|
||||
|
||||
entity.setDelimiter(delimiter);
|
||||
entity.setEncoding(encoding);
|
||||
entity.setNumberOfLines(stats.numberOfLines());
|
||||
entity.setColumnLabels(stats.columnLabels());
|
||||
entity.setVersion(entity.getVersion() + 1);
|
||||
|
||||
componentMappingPersistenceService.updateOrCreate(entity.getStorageId(), mappingFile, entity);
|
||||
|
||||
return mappingEntityMapper.toComponentMappingMetaData(entity);
|
||||
}
|
||||
|
||||
|
||||
private static Charset resolveCharset(String encoding) {
|
||||
|
||||
try {
|
||||
return Charset.forName(encoding);
|
||||
} catch (IllegalCharsetNameException e) {
|
||||
throw new BadRequestException("Invalid character encoding: " + encoding);
|
||||
} catch (UnsupportedCharsetException e) {
|
||||
throw new BadRequestException("Unsupported character encoding: " + encoding);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new BadRequestException("Encoding can't be null.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws CsvException, BadRequestException, IOException {
|
||||
|
||||
Path tempFile = Files.createTempFile("mapping", ".tmp");
|
||||
|
||||
Files.move(mappingFile.toPath(), tempFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
|
||||
String[] columnLabels;
|
||||
int numberOfLines = 0;
|
||||
try (Reader fileReader = new FileReader(tempFile.toFile(), charset);//
|
||||
CSVReader reader = buildReader(fileReader, delimiter);//
|
||||
CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset))) {
|
||||
|
||||
List<String[]> rows = reader.readAll();
|
||||
|
||||
columnLabels = rows.remove(0);
|
||||
|
||||
numberOfLines = (int) reader.getLinesRead() - 1;
|
||||
|
||||
rows.sort(CSV_SORTER);
|
||||
|
||||
writer.writeNext(columnLabels);
|
||||
|
||||
writer.writeAll(rows);
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new BadRequestException("Error while sorting the csv file", e);
|
||||
}
|
||||
|
||||
Files.deleteIfExists(tempFile);
|
||||
|
||||
return new CsvStats(Arrays.asList(columnLabels), numberOfLines);
|
||||
}
|
||||
|
||||
|
||||
private static CSVReader buildReader(Reader reader, char delimiter) throws IOException {
|
||||
|
||||
return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build();
|
||||
}
|
||||
|
||||
|
||||
public void delete(String dossierTemplateId, String componentMappingId) {
|
||||
|
||||
componentMappingPersistenceService.deleteById(dossierTemplateId, componentMappingId);
|
||||
}
|
||||
|
||||
|
||||
public ComponentMappingDownloadModel getMappingForDownload(String dossierTemplateId, String componentMappingId) {
|
||||
|
||||
return componentMappingPersistenceService.getMappingFileForDownload(dossierTemplateId, componentMappingId);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public List<ComponentMapping> getMappingFilesByDossierTemplateId(String dossierTemplateId, Path outputDir) {
|
||||
|
||||
var entities = componentMappingPersistenceService.getByDossierTemplateId(dossierTemplateId);
|
||||
|
||||
return entities.stream()
|
||||
.map(mappingEntityMapper::toComponentMappingMetaData)
|
||||
.map(metaData -> downloadFileAndCreateMapping(outputDir, metaData))
|
||||
.toList();
|
||||
|
||||
}
|
||||
|
||||
|
||||
private ComponentMapping downloadFileAndCreateMapping(Path outputDir, ComponentMappingMetadata metaData) {
|
||||
|
||||
File mappingFile = componentMappingPersistenceService.downloadMappingFileToFolder(metaData.getStorageId(), metaData.getFileName(), outputDir);
|
||||
return new ComponentMapping(metaData, mappingFile);
|
||||
}
|
||||
|
||||
|
||||
public static String buildStorageId(String dossierTemplateId, String id, String name, String fileName) {
|
||||
|
||||
return dossierTemplateId + "/" + id + "_" + name + "_" + fileName;
|
||||
}
|
||||
|
||||
|
||||
private static class CSVSorter implements Comparator<String[]> {
|
||||
|
||||
@Override
|
||||
public int compare(String[] line1, String[] line2) {
|
||||
|
||||
for (int column = 0; column < line1.length; column++) {
|
||||
if (!line1[column].equals(line2[column])) {
|
||||
return line1[column].compareTo(line2[column]);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private record CsvStats(List<String> columnLabels, int numberOfLines) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,5 +1,7 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
@ -10,6 +12,7 @@ import java.util.UUID;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.FileSystemUtils;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
|
||||
@ -20,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
|
||||
@ -40,29 +44,33 @@ import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class DossierTemplateCloneService {
|
||||
|
||||
private final DossierTemplateRepository dossierTemplateRepository;
|
||||
private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
|
||||
private final RulesPersistenceService rulesPersistenceService;
|
||||
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
|
||||
private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
|
||||
private final DictionaryPersistenceService dictionaryPersistenceService;
|
||||
private final EntryPersistenceService entryPersistenceService;
|
||||
private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
|
||||
private final ReportTemplatePersistenceService reportTemplatePersistenceService;
|
||||
private final ColorsService colorsService;
|
||||
private final StorageService storageService;
|
||||
private final DossierStatusPersistenceService dossierStatusPersistenceService;
|
||||
private final WatermarkService watermarkService;
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
DossierTemplateRepository dossierTemplateRepository;
|
||||
LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
|
||||
RulesPersistenceService rulesPersistenceService;
|
||||
DossierTemplatePersistenceService dossierTemplatePersistenceService;
|
||||
DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
|
||||
DictionaryPersistenceService dictionaryPersistenceService;
|
||||
EntryPersistenceService entryPersistenceService;
|
||||
FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
|
||||
ReportTemplatePersistenceService reportTemplatePersistenceService;
|
||||
ColorsService colorsService;
|
||||
StorageService storageService;
|
||||
DossierStatusPersistenceService dossierStatusPersistenceService;
|
||||
WatermarkService watermarkService;
|
||||
FileManagementStorageService fileManagementStorageService;
|
||||
ComponentMappingService componentMappingService;
|
||||
|
||||
|
||||
public DossierTemplateEntity cloneDossierTemplate(String dossierTemplateId, CloneDossierTemplateRequest cloneDossierTemplateRequest) {
|
||||
@ -129,6 +137,8 @@ public class DossierTemplateCloneService {
|
||||
// set the watermark configurations
|
||||
cloneWatermarks(dossierTemplateId, clonedDossierTemplate.getId());
|
||||
|
||||
cloneComponentMappings(dossierTemplateId, clonedDossierTemplate.getId());
|
||||
|
||||
return clonedDossierTemplate;
|
||||
}
|
||||
|
||||
@ -153,6 +163,23 @@ public class DossierTemplateCloneService {
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void cloneComponentMappings(String dossierTemplateId, String clonedDossierTemplateId) {
|
||||
|
||||
Path dir = Files.createTempDirectory("componentMappingsForClone");
|
||||
List<ComponentMapping> componentMappings = componentMappingService.getMappingFilesByDossierTemplateId(dossierTemplateId, dir);
|
||||
for (ComponentMapping componentMapping : componentMappings) {
|
||||
componentMappingService.create(clonedDossierTemplateId,
|
||||
componentMapping.metaData().getName(),
|
||||
componentMapping.metaData().getFileName(),
|
||||
componentMapping.metaData().getDelimiter(),
|
||||
componentMapping.metaData().getEncoding(),
|
||||
componentMapping.file());
|
||||
}
|
||||
FileSystemUtils.deleteRecursively(dir);
|
||||
}
|
||||
|
||||
|
||||
private void cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) {
|
||||
|
||||
var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false);
|
||||
|
||||
@ -8,14 +8,17 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@ -60,6 +63,7 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.FileUtils;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
|
||||
@ -70,6 +74,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ComponentMappingImportModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ExportFilename;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportTemplateResult;
|
||||
@ -81,6 +86,7 @@ import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@ -109,6 +115,7 @@ public class DossierTemplateImportService {
|
||||
private final StorageService storageService;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final FileManagementServiceSettings settings;
|
||||
private final ComponentMappingService componentMappingService;
|
||||
|
||||
|
||||
public String importDossierTemplate(ImportDossierTemplateRequest request) {
|
||||
@ -145,7 +152,8 @@ public class DossierTemplateImportService {
|
||||
Map<String, List<String>> typeEntriesMap = new HashMap<>();
|
||||
Map<String, List<String>> typeFalsePositivesMap = new HashMap<>();
|
||||
Map<String, List<String>> typeFalseRecommendationsMap = new HashMap<>();
|
||||
|
||||
Map<String, byte[]> mappingDataMap = new HashMap<>();
|
||||
List<ComponentMappingMetadata> mappingMetadataList = new LinkedList<>();
|
||||
while ((ze = zis.getNextZipEntry()) != null) {
|
||||
log.debug("---> " + ze.getName() + " ---- " + ze.isDirectory());
|
||||
totalEntryArchive++;
|
||||
@ -274,12 +282,22 @@ public class DossierTemplateImportService {
|
||||
reportTemplateFilenameList = reportTemplateList.stream()
|
||||
.map(rt -> rt.isMultiFileReport() ? rt.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : rt.getFileName())
|
||||
.toList();
|
||||
} else if (ze.getName().startsWith(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename())) {
|
||||
if (ze.getName().contains(ExportFilename.COMPONENT_MAPPINGS_FILE.getFilename())) {
|
||||
mappingMetadataList = objectMapper.readValue(bytes, new TypeReference<>() {
|
||||
});
|
||||
} else if (ze.getName().endsWith(".csv")) {
|
||||
String fileName = ze.getName().replace(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename() + "/", "");
|
||||
mappingDataMap.put(fileName, bytes);
|
||||
}
|
||||
|
||||
} else {
|
||||
reportTemplateBytesMap.put(ze.getName(), bos);
|
||||
}
|
||||
bos.close();
|
||||
}
|
||||
}
|
||||
|
||||
importTemplateResult.setEntries(typeEntriesMap);
|
||||
importTemplateResult.setFalsePositives(typeFalsePositivesMap);
|
||||
importTemplateResult.setFalseRecommendations(typeFalseRecommendationsMap);
|
||||
@ -298,6 +316,13 @@ public class DossierTemplateImportService {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
for (var metadata : mappingMetadataList) {
|
||||
String fileName = metadata.getName() + ".csv";
|
||||
if (mappingDataMap.containsKey(fileName)) {
|
||||
importTemplateResult.componentMappings.add(new ComponentMappingImportModel(metadata, mappingDataMap.get(fileName)));
|
||||
}
|
||||
}
|
||||
if (importTemplateResult.getDossierTemplate() == null) {
|
||||
throw new BadRequestException("Provided archive is faulty");
|
||||
}
|
||||
@ -335,9 +360,6 @@ public class DossierTemplateImportService {
|
||||
updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId());
|
||||
dossierTemplateRepository.save(existingDossierTemplate);
|
||||
|
||||
// set rules
|
||||
setRulesWhenCompiled(request, dossierTemplateId);
|
||||
|
||||
existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(existingDossierTemplate)
|
||||
.name()));
|
||||
|
||||
@ -448,12 +470,11 @@ public class DossierTemplateImportService {
|
||||
dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
dossierTemplateEntity.setCreatedBy(request.getUserId());
|
||||
//set rules
|
||||
setRulesWhenCompiled(request, dossierTemplateEntity.getId());
|
||||
|
||||
var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity);
|
||||
loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate));
|
||||
dossierTemplateId = loadedDossierTemplate.getId();
|
||||
|
||||
dossierTemplateId = loadedDossierTemplate.getId();
|
||||
// set colors
|
||||
this.setColors(dossierTemplateId, request.getColors());
|
||||
|
||||
@ -527,17 +548,44 @@ public class DossierTemplateImportService {
|
||||
FileAttributesGeneralConfigurationEntity.class));
|
||||
}
|
||||
|
||||
setRulesWhenCompiled(request, dossierTemplateId);
|
||||
setComponentMappings(dossierTemplateId, request.getComponentMappings());
|
||||
|
||||
long elapsedTime = System.currentTimeMillis() - start;
|
||||
log.info("stop import dossier template elapsedTime: " + elapsedTime + "for: " + dossierTemplateId);
|
||||
return dossierTemplateId;
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void setComponentMappings(String dossierTemplateId, List<ComponentMappingImportModel> componentMappings) {
|
||||
|
||||
List<ComponentMappingMetadata> existingMappings = componentMappingService.getMetaDataByDossierTemplateId(dossierTemplateId);
|
||||
existingMappings.forEach(metadata -> componentMappingService.delete(dossierTemplateId, metadata.getId()));
|
||||
|
||||
for (ComponentMappingImportModel componentMapping : componentMappings) {
|
||||
File tmpFile = Files.createTempFile("mapping", ".csv").toFile();
|
||||
try (var out = new FileOutputStream(tmpFile)) {
|
||||
out.write(componentMapping.csvData());
|
||||
}
|
||||
componentMappingService.create(dossierTemplateId,
|
||||
componentMapping.metadata().getName(),
|
||||
componentMapping.metadata().getFileName(),
|
||||
componentMapping.metadata().getDelimiter(),
|
||||
componentMapping.metadata().getEncoding(),
|
||||
tmpFile);
|
||||
assert tmpFile.delete();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void setRulesWhenCompiled(ImportTemplateResult request, String dossierTemplateId) {
|
||||
|
||||
DroolsValidation droolsValidation = rulesValidationService.validateRules(RuleFileType.ENTITY, request.getRuleSet());
|
||||
if (!droolsValidation.isCompiled()) {
|
||||
droolsValidation.getSyntaxErrorMessages().forEach(errorMessage -> log.error(errorMessage.getMessage()));
|
||||
droolsValidation.getSyntaxErrorMessages()
|
||||
.forEach(errorMessage -> log.error(errorMessage.getMessage()));
|
||||
droolsValidation.getBlacklistErrorMessages()
|
||||
.forEach(errorMessage -> log.error(errorMessage.getMessage()));
|
||||
throw new BadRequestException("The entity rules do not compile or contain blacklisted keywords!");
|
||||
@ -546,7 +594,8 @@ public class DossierTemplateImportService {
|
||||
if (request.getComponentRuleSet() != null) {
|
||||
DroolsValidation componentDroolsValidation = rulesValidationService.validateRules(RuleFileType.COMPONENT, request.getComponentRuleSet());
|
||||
if (!componentDroolsValidation.isCompiled()) {
|
||||
componentDroolsValidation.getSyntaxErrorMessages().forEach(errorMessage -> log.error(errorMessage.getMessage()));
|
||||
componentDroolsValidation.getSyntaxErrorMessages()
|
||||
.forEach(errorMessage -> log.error(errorMessage.getMessage()));
|
||||
componentDroolsValidation.getBlacklistErrorMessages()
|
||||
.forEach(errorMessage -> log.error(errorMessage.getMessage()));
|
||||
throw new BadRequestException("The component rules do not compile or contain blacklisted keywords!");
|
||||
|
||||
@ -89,7 +89,7 @@ public class DownloadService {
|
||||
request.getReportTemplateIds(),
|
||||
request.getRedactionPreviewColor());
|
||||
websocketService.sendDownloadEvent(storageId, request.getUserId(), DownloadStatusValue.QUEUED);
|
||||
addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).includeUnprocessed(request.getIncludeUnprocessed()).build(), 1);
|
||||
addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).build(), 1);
|
||||
|
||||
return new JSONPrimitive<>(storageId);
|
||||
}
|
||||
|
||||
@ -66,6 +66,7 @@ public class FileStatusMapper {
|
||||
.lastIndexed(status.getLastIndexed())
|
||||
.fileSize(status.getFileSize())
|
||||
.fileErrorInfo(status.getFileErrorInfo())
|
||||
.componentMappingVersions(status.getComponentMappingVersions())
|
||||
.build();
|
||||
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
|
||||
@ -39,17 +40,11 @@ public class FileStatusProcessingUpdateService {
|
||||
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
|
||||
|
||||
switch (analyzeResult.getMessageType()) {
|
||||
|
||||
case SURROUNDING_TEXT_ANALYSIS:
|
||||
break;
|
||||
case ANALYSE:
|
||||
case REANALYSE:
|
||||
default:
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
|
||||
fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult);
|
||||
return null;
|
||||
});
|
||||
log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, 0);
|
||||
fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult);
|
||||
|
||||
if (!analyzeResult.isWasReanalyzed()) {
|
||||
indexingService.addToIndexingQueue(IndexMessageType.INSERT, dossier.getDossierTemplateId(), dossierId, fileId, 2);
|
||||
@ -60,6 +55,7 @@ public class FileStatusProcessingUpdateService {
|
||||
if (analyzeResult.getAddedFileAttributes() != null && !analyzeResult.getAddedFileAttributes().isEmpty()) {
|
||||
fileStatusPersistenceService.addFileAttributes(dossierId, fileId, analyzeResult.getAddedFileAttributes());
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
@ -79,16 +75,16 @@ public class FileStatusProcessingUpdateService {
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.info("Preprocessing dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
|
||||
fileStatusService.setStatusPreProcessing(fileId,
|
||||
fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
|
||||
fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
|
||||
return null;
|
||||
});
|
||||
|
||||
var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId);
|
||||
if (updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()) {
|
||||
throw new ConflictException(String.format("Max Processing Retries exhausted for dossier %s and file %s with errorCount: %s",
|
||||
dossierId,
|
||||
fileId,
|
||||
updatedFileEntity.getProcessingErrorCounter()));
|
||||
dossierId,
|
||||
fileId,
|
||||
updatedFileEntity.getProcessingErrorCounter()));
|
||||
}
|
||||
}
|
||||
|
||||
@ -106,10 +102,10 @@ public class FileStatusProcessingUpdateService {
|
||||
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ",
|
||||
retryContext.getRetryCount(),
|
||||
fileId,
|
||||
dossierId,
|
||||
fileErrorInfo != null ? fileErrorInfo.getCause() : null);
|
||||
retryContext.getRetryCount(),
|
||||
fileId,
|
||||
dossierId,
|
||||
fileErrorInfo != null ? fileErrorInfo.getCause() : null);
|
||||
fileStatusService.setStatusError(dossierId, fileId, fileErrorInfo);
|
||||
return null;
|
||||
});
|
||||
@ -123,7 +119,7 @@ public class FileStatusProcessingUpdateService {
|
||||
ocrFailed(dossierId, fileId, fileErrorInfo);
|
||||
} else {
|
||||
fileStatusService.setStatusOcrProcessing(fileId,
|
||||
fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
|
||||
fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
|
||||
fileStatusService.addToOcrQueue(dossierId, fileId, 2);
|
||||
}
|
||||
}
|
||||
@ -136,7 +132,7 @@ public class FileStatusProcessingUpdateService {
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.info("Ocr processing dossier {} and file {}, Attempt to update status: {}", fileEntity.getDossierId(), fileId, retryContext.getRetryCount());
|
||||
fileStatusService.setStatusOcrProcessing(fileId,
|
||||
fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
|
||||
fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
@ -149,9 +145,9 @@ public class FileStatusProcessingUpdateService {
|
||||
var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId);
|
||||
if (updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()) {
|
||||
throw new ConflictException(String.format("Max Ocr Retries exhausted for dossier %s and file %s with errorCount: %s",
|
||||
updatedFileEntity.getDossierId(),
|
||||
fileId,
|
||||
updatedFileEntity.getProcessingErrorCounter()));
|
||||
updatedFileEntity.getDossierId(),
|
||||
fileId,
|
||||
updatedFileEntity.getProcessingErrorCounter()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
@ -90,6 +91,8 @@ public class FileStatusService {
|
||||
ViewedPagesPersistenceService viewedPagesPersistenceService;
|
||||
FileManagementServiceSettings fileManagementServiceSettings;
|
||||
LayoutParsingRequestFactory layoutParsingRequestFactory;
|
||||
ComponentMappingService componentMappingService;
|
||||
|
||||
WebsocketService websocketService;
|
||||
|
||||
@Transactional
|
||||
@ -252,6 +255,7 @@ public class FileStatusService {
|
||||
.sectionsToReanalyse(sectionsToReanalyse)
|
||||
.fileId(fileId)
|
||||
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
|
||||
.componentMappings(componentMappingService.getMetaDataByDossierTemplateId(dossierTemplate.getId()))
|
||||
.dossierTemplateId(dossier.getDossierTemplateId())
|
||||
.lastProcessed(fileModel.getLastProcessed())
|
||||
.fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
|
||||
@ -517,7 +521,8 @@ public class FileStatusService {
|
||||
analyzeResult.getDuration(),
|
||||
analyzeResult.getDossierDictionaryVersion(),
|
||||
analyzeResult.getAnalysisVersion(),
|
||||
analyzeResult.getAnalysisNumber());
|
||||
analyzeResult.getAnalysisNumber(),
|
||||
analyzeResult.getUsedComponentMappings());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -7,8 +7,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.service.
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@ -19,23 +22,28 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class ReanalysisRequiredStatusService {
|
||||
|
||||
private final DictionaryPersistenceService dictionaryPersistenceService;
|
||||
private final RulesPersistenceService rulesPersistenceService;
|
||||
private final DossierPersistenceService dossierPersistenceService;
|
||||
private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
|
||||
DictionaryPersistenceService dictionaryPersistenceService;
|
||||
RulesPersistenceService rulesPersistenceService;
|
||||
DossierPersistenceService dossierPersistenceService;
|
||||
LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
|
||||
ComponentMappingService componentMappingService;
|
||||
|
||||
|
||||
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) {
|
||||
@ -78,8 +86,12 @@ public class ReanalysisRequiredStatusService {
|
||||
fileStatus.setDossierTemplateId(dossier.getDossierTemplateId());
|
||||
fileStatus.setDossierStatusId(dossier.getDossierStatusId());
|
||||
|
||||
if (fileStatus.isSoftOrHardDeleted()) {
|
||||
log.debug("File {} is deleted, thus analysis is not required", fileStatus.getId());
|
||||
return new AnalysisRequiredResult(false, false);
|
||||
}
|
||||
if (dossier.getSoftDeletedTime() != null || dossier.getHardDeletedTime() != null || dossier.getArchivedTime() != null) {
|
||||
log.info("Dossier {} is deleted, thus analysis is not required", fileStatus.getDossierId());
|
||||
log.debug("Dossier {} is deleted, thus analysis is not required", fileStatus.getDossierId());
|
||||
return new AnalysisRequiredResult(false, false);
|
||||
}
|
||||
|
||||
@ -118,39 +130,141 @@ public class ReanalysisRequiredStatusService {
|
||||
Map<String, Long> dossierVersionMap) {
|
||||
|
||||
// get relevant versions
|
||||
var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), k -> buildVersionData(dossier.getDossierTemplateId()));
|
||||
var dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId()));
|
||||
Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
|
||||
k -> buildVersionData(dossier.getDossierTemplateId()));
|
||||
|
||||
Map<String, Integer> componentMappingVersions = componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId())
|
||||
.stream()
|
||||
.collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion));
|
||||
|
||||
Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId()));
|
||||
|
||||
// compute matches
|
||||
var mappingVersionAllMatch = mappingVersionsEqual(fileStatus, componentMappingVersions);
|
||||
var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L);
|
||||
var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L);
|
||||
var dictionaryVersionMatches = fileStatus.getDictionaryVersion() == dossierTemplateVersions.getOrDefault(DICTIONARY, -1L);
|
||||
var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L);
|
||||
var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion;
|
||||
|
||||
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches;
|
||||
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch;
|
||||
var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches;
|
||||
|
||||
if (reanalysisRequired || fullAnalysisRequired) {
|
||||
log.info(
|
||||
"For file: {}-{} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}",
|
||||
fileStatus.getId(),
|
||||
fileStatus.getFilename(),
|
||||
fileStatus.getRulesVersion(),
|
||||
dossierTemplateVersions.getOrDefault(RULES, -1L),
|
||||
fileStatus.getComponentRulesVersion(),
|
||||
dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L),
|
||||
fileStatus.getDictionaryVersion(),
|
||||
dossierTemplateVersions.getOrDefault(DICTIONARY, -1L),
|
||||
fileStatus.getLegalBasisVersion(),
|
||||
dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L),
|
||||
Math.max(fileStatus.getDossierDictionaryVersion(), 0),
|
||||
dossierDictionaryVersion);
|
||||
|
||||
log.info(buildMessage(fileStatus,
|
||||
rulesVersionMatches,
|
||||
dossierTemplateVersions,
|
||||
componentRulesVersionMatches,
|
||||
dictionaryVersionMatches,
|
||||
legalBasisVersionMatches,
|
||||
dossierDictionaryVersionMatches,
|
||||
dossierDictionaryVersion,
|
||||
mappingVersionAllMatch,
|
||||
componentMappingVersions));
|
||||
}
|
||||
return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private String buildMessage(FileModel fileStatus,
|
||||
boolean rulesVersionMatches,
|
||||
Map<VersionType, Long> dossierTemplateVersions,
|
||||
boolean componentRulesVersionMatches,
|
||||
boolean dictionaryVersionMatches,
|
||||
boolean legalBasisVersionMatches,
|
||||
boolean dossierDictionaryVersionMatches,
|
||||
Long dossierDictionaryVersion,
|
||||
boolean mappingVersionAllMatch,
|
||||
Map<String, Integer> componentMappingVersions) {
|
||||
|
||||
StringBuilder messageBuilder = new StringBuilder();
|
||||
messageBuilder.append("For file: ").append(fileStatus.getId()).append("-").append(fileStatus.getFilename()).append(" analysis is required because -> ");
|
||||
|
||||
boolean needComma = false;
|
||||
|
||||
if (!rulesVersionMatches) {
|
||||
messageBuilder.append("ruleVersions: ").append(fileStatus.getRulesVersion()).append("/").append(dossierTemplateVersions.getOrDefault(RULES, -1L));
|
||||
needComma = true;
|
||||
}
|
||||
|
||||
if (!componentRulesVersionMatches) {
|
||||
if (needComma) {
|
||||
messageBuilder.append(", ");
|
||||
}
|
||||
messageBuilder.append("componentRuleVersions: ")
|
||||
.append(fileStatus.getComponentRulesVersion())
|
||||
.append("/")
|
||||
.append(dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L));
|
||||
needComma = true;
|
||||
}
|
||||
|
||||
if (!dictionaryVersionMatches) {
|
||||
if (needComma) {
|
||||
messageBuilder.append(", ");
|
||||
}
|
||||
messageBuilder.append("dictionaryVersions: ").append(fileStatus.getDictionaryVersion()).append("/").append(dossierTemplateVersions.getOrDefault(DICTIONARY, -1L));
|
||||
needComma = true;
|
||||
}
|
||||
|
||||
if (!legalBasisVersionMatches) {
|
||||
if (needComma) {
|
||||
messageBuilder.append(", ");
|
||||
}
|
||||
messageBuilder.append("legalBasisVersions: ").append(fileStatus.getLegalBasisVersion()).append("/").append(dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L));
|
||||
needComma = true;
|
||||
}
|
||||
|
||||
if (!dossierDictionaryVersionMatches) {
|
||||
if (needComma) {
|
||||
messageBuilder.append(", ");
|
||||
}
|
||||
messageBuilder.append("dossierDictionaryVersions: ").append(Math.max(fileStatus.getDossierDictionaryVersion(), 0)).append("/").append(dossierDictionaryVersion);
|
||||
needComma = true;
|
||||
}
|
||||
|
||||
if (!mappingVersionAllMatch) {
|
||||
if (needComma) {
|
||||
messageBuilder.append(", ");
|
||||
}
|
||||
messageBuilder.append("componentMappingVersions: ").append(buildMappingVersionMatchesString(fileStatus.getComponentMappingVersions(), componentMappingVersions));
|
||||
needComma = true;
|
||||
}
|
||||
|
||||
return messageBuilder.toString();
|
||||
}
|
||||
|
||||
|
||||
private String buildMappingVersionMatchesString(Map<String, Integer> fileComponentMappingVersions, Map<String, Integer> dbComponentMappingVersions) {
|
||||
|
||||
Set<String> allMappingNames = new HashSet<>();
|
||||
allMappingNames.addAll(fileComponentMappingVersions.keySet());
|
||||
allMappingNames.addAll(dbComponentMappingVersions.keySet());
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
allMappingNames.stream()
|
||||
.sorted()
|
||||
.forEach(mappingName -> {
|
||||
long fileVersion = fileComponentMappingVersions.getOrDefault(mappingName, -1);
|
||||
long dbVersion = dbComponentMappingVersions.getOrDefault(mappingName, -1);
|
||||
if (fileVersion != dbVersion) {
|
||||
sb.append(mappingName).append(": ").append(fileVersion).append("/").append(dbVersion);
|
||||
}
|
||||
});
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
private static boolean mappingVersionsEqual(FileModel fileStatus, Map<String, Integer> componentMappingVersions) {
|
||||
|
||||
return fileStatus.getComponentMappingVersions().keySet().equals(componentMappingVersions.keySet()) && fileStatus.getComponentMappingVersions().keySet()
|
||||
.stream()
|
||||
.allMatch(name -> fileStatus.getComponentMappingVersions()
|
||||
.get(name).equals(componentMappingVersions.get(name)));
|
||||
}
|
||||
|
||||
|
||||
private Map<VersionType, Long> buildVersionData(String dossierTemplateId) {
|
||||
|
||||
var versions = new HashMap<VersionType, Long>();
|
||||
|
||||
@ -4,16 +4,20 @@ import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicC
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.FileSystemUtils;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
@ -24,8 +28,10 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
@ -43,6 +49,7 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.FileAttributesGeneralConfiguration;
|
||||
@ -79,6 +86,7 @@ public class DossierTemplateExportService {
|
||||
private final RulesPersistenceService rulesPersistenceService;
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
private final ReportTemplatePersistenceService reportTemplatePersistenceService;
|
||||
private final ComponentMappingService componentMappingService;
|
||||
private final ColorsService colorsService;
|
||||
private final EntryPersistenceService entryPersistenceService;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
@ -117,6 +125,7 @@ public class DossierTemplateExportService {
|
||||
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId());
|
||||
downloadStatusPersistenceService.updateStatus(downloadJob.getStorageId(), DownloadStatusValue.GENERATING);
|
||||
|
||||
String dossierTemplateId = extractDossierTemplateId(downloadStatus.getFilename());
|
||||
var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId);
|
||||
@ -234,6 +243,26 @@ public class DossierTemplateExportService {
|
||||
writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(), getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT));
|
||||
}
|
||||
|
||||
Path mappingDir = Files.createTempDirectory("mappings");
|
||||
|
||||
List<ComponentMapping> componentMappings = componentMappingService.getMappingFilesByDossierTemplateId(dossierTemplateId, mappingDir);
|
||||
|
||||
List<ComponentMappingMetadata> metadata = componentMappings.stream()
|
||||
.map(ComponentMapping::metaData)
|
||||
.toList();
|
||||
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename(),
|
||||
ExportFilename.COMPONENT_MAPPINGS_FILE.getFilename() + JSON_EXT,
|
||||
objectMapper.writeValueAsBytes(metadata)));
|
||||
for (ComponentMapping componentMapping : componentMappings) {
|
||||
try (var in = new FileInputStream(componentMapping.file())) {
|
||||
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename(),
|
||||
componentMapping.metaData().getName() + ".csv",
|
||||
in.readAllBytes()));
|
||||
}
|
||||
}
|
||||
|
||||
FileSystemUtils.deleteRecursively(mappingDir);
|
||||
|
||||
storeZipFile(downloadStatus.getStorageId(), fileSystemBackedArchiver);
|
||||
downloadStatusPersistenceService.updateStatus(downloadStatus, DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.job;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@ -69,7 +70,9 @@ public class AutomaticAnalysisJob implements Job {
|
||||
if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) {
|
||||
log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId());
|
||||
return;
|
||||
}var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
|
||||
}
|
||||
|
||||
var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
|
||||
if (redactionQueueInfo != null) {
|
||||
log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
|
||||
tenant.getTenantId(),
|
||||
@ -80,17 +83,17 @@ public class AutomaticAnalysisJob implements Job {
|
||||
var consumerCount = redactionQueueInfo.getConsumerCount();
|
||||
if (redactionQueueInfo.getMessageCount() <= consumerCount * 5) {
|
||||
// queue up 5 files
|
||||
var allStatuses = getAllRelevantStatuses();
|
||||
List<FileModel> allStatuses = getAllRelevantStatuses();
|
||||
|
||||
allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated));
|
||||
|
||||
var allStatusesIterator = allStatuses.iterator();
|
||||
Iterator<FileModel> allStatusesIterator = allStatuses.iterator();
|
||||
log.debug("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size());
|
||||
|
||||
var queuedFiles = 0;
|
||||
int queuedFiles = 0;
|
||||
|
||||
while (queuedFiles < (consumerCount * 5) && allStatusesIterator.hasNext()) {
|
||||
var next = allStatusesIterator.next();
|
||||
FileModel next = allStatusesIterator.next();
|
||||
// in case the file doesn't have numberOfPages set, we assume an average.
|
||||
|
||||
reanalyseFile(next);
|
||||
|
||||
@ -2,9 +2,12 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@ -14,6 +17,7 @@ import org.springframework.stereotype.Service;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier;
|
||||
@ -23,10 +27,12 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Websock
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
|
||||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -42,6 +48,7 @@ public class FileStatusPersistenceService {
|
||||
private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
|
||||
private final DossierPersistenceService dossierService;
|
||||
private final WebsocketService websocketService;
|
||||
private final EntityManager entityManager;
|
||||
|
||||
|
||||
public void createStatus(String dossierId, String fileId, String filename, String uploader, long size) {
|
||||
@ -61,6 +68,7 @@ public class FileStatusPersistenceService {
|
||||
file.setFileManipulationDate(now);
|
||||
file.setProcessingErrorCounter(0);
|
||||
file.setFileSize(size);
|
||||
file.setComponentMappingVersions(new ArrayList<>());
|
||||
fileRepository.save(file);
|
||||
}
|
||||
|
||||
@ -91,21 +99,15 @@ public class FileStatusPersistenceService {
|
||||
|
||||
private int calculateProcessingErrorCounter(String fileId, ProcessingStatus processingStatus) {
|
||||
|
||||
switch (processingStatus) {
|
||||
case ERROR:
|
||||
return fileRepository.findById(fileId)
|
||||
.map(FileEntity::getProcessingErrorCounter)
|
||||
.orElse(0) + 1;
|
||||
|
||||
case PROCESSED:
|
||||
case REPROCESS:
|
||||
return 0;
|
||||
|
||||
default:
|
||||
return fileRepository.findById(fileId)
|
||||
.map(FileEntity::getProcessingErrorCounter)
|
||||
.orElse(0);
|
||||
}
|
||||
return switch (processingStatus) {
|
||||
case ERROR -> fileRepository.findById(fileId)
|
||||
.map(FileEntity::getProcessingErrorCounter)
|
||||
.orElse(0) + 1;
|
||||
case PROCESSED, REPROCESS -> 0;
|
||||
default -> fileRepository.findById(fileId)
|
||||
.map(FileEntity::getProcessingErrorCounter)
|
||||
.orElse(0);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -121,11 +123,13 @@ public class FileStatusPersistenceService {
|
||||
long duration,
|
||||
long dossierDictionaryVersion,
|
||||
int analysisVersion,
|
||||
int analysisNumber) {
|
||||
int analysisNumber,
|
||||
List<ComponentMappingMetadata> usedComponentMappings) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
fileRepository.updateProcessingStatus(fileId,
|
||||
numberOfPages,
|
||||
ProcessingStatus.PROCESSED,
|
||||
@ -141,10 +145,30 @@ public class FileStatusPersistenceService {
|
||||
analysisNumber,
|
||||
calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
|
||||
|
||||
// must be modifiable, otherwise hibernate fails
|
||||
List<FileEntityComponentMappingVersionEntity> versionEntities = getFileEntityComponentMappingVersionEntities(usedComponentMappings);
|
||||
|
||||
FileEntity fileEntity = entityManager.find(FileEntity.class, fileId);
|
||||
|
||||
fileEntity.setComponentMappingVersions(versionEntities);
|
||||
|
||||
entityManager.merge(fileEntity);
|
||||
|
||||
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.FINISHED, analysisNumber);
|
||||
}
|
||||
|
||||
|
||||
private static List<FileEntityComponentMappingVersionEntity> getFileEntityComponentMappingVersionEntities(List<ComponentMappingMetadata> usedComponentMappings) {
|
||||
|
||||
if (usedComponentMappings == null) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
return usedComponentMappings.stream()
|
||||
.map(cm -> new FileEntityComponentMappingVersionEntity(cm.getName(), cm.getVersion()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates) {
|
||||
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DigitalSignatureEntity;
|
||||
|
||||
public interface ComponentMappingRepository extends JpaRepository<ComponentMappingEntity, String> {
|
||||
|
||||
List<ComponentMappingEntity> findByDossierTemplateId(String dossierTemplateId);
|
||||
|
||||
|
||||
boolean existsByNameAndDossierTemplateId(String name, String dossierTemplateId);
|
||||
|
||||
|
||||
Optional<ComponentMappingEntity> findByIdAndDossierTemplateId(String id, String dossierTemplateId);
|
||||
|
||||
}
|
||||
@ -47,12 +47,21 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, "
|
||||
+ "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, "
|
||||
+ "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, "
|
||||
+ "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, "
|
||||
+ "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter "
|
||||
+ "where f.id = :fileId")
|
||||
@Query("""
|
||||
update FileEntity f set f.numberOfPages = :numberOfPages, \
|
||||
f.processingStatus = :processingStatus, \
|
||||
f.dictionaryVersion = :dictionaryVersion, \
|
||||
f.rulesVersion = :rulesVersion, \
|
||||
f.componentRulesVersion = :componentRulesVersion, \
|
||||
f.legalBasisVersion = :legalBasisVersion, \
|
||||
f.analysisDuration = :analysisDuration, \
|
||||
f.dossierDictionaryVersion = :dossierDictionaryVersion, \
|
||||
f.analysisVersion = :analysisVersion, \
|
||||
f.numberOfAnalyses = :analysisNumber, \
|
||||
f.lastUpdated = :lastUpdated, \
|
||||
f.lastProcessed = :lastProcessed, \
|
||||
f.processingErrorCounter = :processingErrorCounter \
|
||||
where f.id = :fileId""")
|
||||
void updateProcessingStatus(@Param("fileId") String fileId,
|
||||
@Param("numberOfPages") int numberOfPages,
|
||||
@Param("processingStatus") ProcessingStatus processingStatus,
|
||||
@ -168,12 +177,13 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
||||
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
||||
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id in (:fileIds)")
|
||||
int softDeleteFiles(@Param("fileIds") List<String> fileIds,
|
||||
@Param("processingStatus") ProcessingStatus processingStatus,
|
||||
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
||||
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
|
||||
@Param("processingStatus") ProcessingStatus processingStatus,
|
||||
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
||||
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
|
||||
|
||||
|
||||
@Modifying
|
||||
@ -380,10 +390,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
||||
+ " when f.deleted is not null then f.deleted "
|
||||
+ "end "
|
||||
+ "where f.id in (:fileIds)")
|
||||
int hardDeleteFiles(@Param("fileIds") List<String> fileIds,
|
||||
@Param("processingStatus") ProcessingStatus processingStatus,
|
||||
@Param("deletionTime") OffsetDateTime deletionTime);
|
||||
|
||||
int hardDeleteFiles(@Param("fileIds") List<String> fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("deletionTime") OffsetDateTime deletionTime);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -53,7 +53,7 @@ public class OCRProcessingMessageReceiver {
|
||||
response.getNumberOfOCRedPages());
|
||||
}
|
||||
|
||||
log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE);
|
||||
log.debug("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE);
|
||||
}
|
||||
|
||||
|
||||
@ -63,7 +63,7 @@ public class OCRProcessingMessageReceiver {
|
||||
|
||||
var response = objectMapper.readValue(failedMessage.getBody(), OCRStatusUpdateResponse.class);
|
||||
|
||||
log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL);
|
||||
log.debug("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,7 +1,12 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.utils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
@ -14,6 +19,18 @@ public class FileModelMapper implements BiConsumer<FileEntity, FileModel> {
|
||||
fileEntity.getFileAttributes()
|
||||
.forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue()));
|
||||
fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp()));
|
||||
fileModel.setComponentMappingVersions(getComponentMappingVersions(fileEntity));
|
||||
}
|
||||
|
||||
|
||||
private static Map<String, Integer> getComponentMappingVersions(FileEntity fileEntity) {
|
||||
|
||||
if (Objects.isNull(fileEntity.getComponentMappingVersions())) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
return fileEntity.getComponentMappingVersions()
|
||||
.stream()
|
||||
.collect(Collectors.toMap(FileEntityComponentMappingVersionEntity::getName, FileEntityComponentMappingVersionEntity::getVersion));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -202,4 +202,8 @@ databaseChangeLog:
|
||||
- include:
|
||||
file: db/changelog/tenant/125-add-max-size-for-legal-basis-in-manual-legal-basis-change.yaml
|
||||
- include:
|
||||
file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml
|
||||
file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml
|
||||
- include:
|
||||
file: db/changelog/tenant/127-add-component-mapping-table.yaml
|
||||
- include:
|
||||
file: db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml
|
||||
|
||||
@ -0,0 +1,97 @@
|
||||
databaseChangeLog:
|
||||
- changeSet:
|
||||
id: create-table-component_mappings
|
||||
author: kilian
|
||||
changes:
|
||||
- createTable:
|
||||
tableName: component_mappings
|
||||
columns:
|
||||
- column:
|
||||
name: id
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
primaryKey: true
|
||||
- column:
|
||||
name: dossier_template_id
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
- column:
|
||||
name: storage_id
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
- column:
|
||||
name: name
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
- column:
|
||||
name: file_name
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
- column:
|
||||
name: version
|
||||
type: integer
|
||||
constraints:
|
||||
nullable: false
|
||||
defaultValue: '-1'
|
||||
- column:
|
||||
name: changed_date
|
||||
type: TIMESTAMP WITH TIME ZONE
|
||||
constraints:
|
||||
nullable: false
|
||||
- column:
|
||||
name: number_of_lines
|
||||
type: INTEGER
|
||||
constraints:
|
||||
nullable: false
|
||||
defaultValue: '0'
|
||||
- column:
|
||||
name: encoding
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
defaultValue: 'UTF-8'
|
||||
- column:
|
||||
name: delimiter
|
||||
type: CHAR(1)
|
||||
constraints:
|
||||
nullable: false
|
||||
defaultValue: ','
|
||||
- addUniqueConstraint:
|
||||
columnNames: name, dossier_template_id
|
||||
constraintName: unique_name_dossier_template
|
||||
tableName: component_mappings
|
||||
- addForeignKeyConstraint:
|
||||
baseColumnNames: dossier_template_id
|
||||
baseTableName: component_mappings
|
||||
constraintName: fk_entity_dossier_template_id
|
||||
referencedColumnNames: id
|
||||
referencedTableName: dossier_template
|
||||
onDelete: CASCADE
|
||||
onUpdate: CASCADE
|
||||
|
||||
- createTable:
|
||||
tableName: component_mapping_column_labels
|
||||
columns:
|
||||
- column:
|
||||
name: component_mapping_entity_id
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
- column:
|
||||
name: column_labels
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
|
||||
- addForeignKeyConstraint:
|
||||
baseColumnNames: component_mapping_entity_id
|
||||
baseTableName: component_mapping_column_labels
|
||||
constraintName: fk_component_mapping_column_labels_entity_id
|
||||
referencedColumnNames: id
|
||||
referencedTableName: component_mappings
|
||||
onDelete: CASCADE
|
||||
onUpdate: CASCADE
|
||||
@ -0,0 +1,31 @@
|
||||
databaseChangeLog:
|
||||
- changeSet:
|
||||
id: add_component_mapping_version_to_file_entity
|
||||
author: kilian
|
||||
changes:
|
||||
- createTable:
|
||||
tableName: file_entity_component_mapping_versions
|
||||
columns:
|
||||
- column:
|
||||
name: file_entity_id
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
- column:
|
||||
name: name
|
||||
type: VARCHAR(255)
|
||||
constraints:
|
||||
nullable: false
|
||||
- column:
|
||||
name: version
|
||||
type: INT
|
||||
constraints:
|
||||
nullable: false
|
||||
- addForeignKeyConstraint:
|
||||
baseColumnNames: file_entity_id
|
||||
baseTableName: file_entity_component_mapping_versions
|
||||
constraintName: fk_component_mapping_version_entity_file
|
||||
referencedColumnNames: id
|
||||
referencedTableName: file
|
||||
onDelete: CASCADE
|
||||
onUpdate: CASCADE
|
||||
@ -27,6 +27,7 @@ dependencies {
|
||||
api("junit:junit:4.13.2")
|
||||
api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
|
||||
api("net.logstash.logback:logstash-logback-encoder:7.4")
|
||||
|
||||
testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
|
||||
testImplementation("org.springframework.security:spring-security-test:6.0.2")
|
||||
testImplementation("org.testcontainers:postgresql:1.17.1")
|
||||
|
||||
@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
@ -115,6 +116,7 @@ public class FileTesterAndProvider {
|
||||
result.setDossierDictionaryVersion(1);
|
||||
result.setAnalysisNumber(1);
|
||||
result.setAnalysisVersion(1);
|
||||
result.setUsedComponentMappings(Collections.emptyList());
|
||||
fileStatusService.setStatusSuccessful(dossierId, fileId, result);
|
||||
fileStatusService.setStatusProcessed(fileId);
|
||||
}
|
||||
|
||||
@ -10,10 +10,6 @@ import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.security.test.context.support.WithMockUser;
|
||||
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
|
||||
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.CustomPermissionClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
|
||||
|
||||
@ -26,6 +26,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
|
||||
@ -73,6 +74,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
|
||||
@MockBean
|
||||
private ColorsService colorsService;
|
||||
@MockBean
|
||||
private ComponentMappingService componentMappingService;
|
||||
@MockBean
|
||||
private StorageService storageService;
|
||||
@MockBean
|
||||
private DossierStatusPersistenceService dossierStatusPersistenceService;
|
||||
@ -125,7 +128,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
|
||||
storageService,
|
||||
dossierStatusPersistenceService,
|
||||
watermarkService,
|
||||
fileManagementStorageService);
|
||||
fileManagementStorageService,
|
||||
componentMappingService);
|
||||
dossierTemplateExportService = new DossierTemplateExportService(dossierTemplatePersistenceService,
|
||||
downloadStatusPersistenceService,
|
||||
dossierAttributeConfigPersistenceService,
|
||||
@ -136,6 +140,7 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
|
||||
rulesPersistenceService,
|
||||
fileManagementStorageService,
|
||||
reportTemplatePersistenceService,
|
||||
componentMappingService,
|
||||
colorsService,
|
||||
entryPersistenceService,
|
||||
watermarkService,
|
||||
|
||||
@ -28,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
|
||||
@ -91,6 +92,8 @@ public class DossierTemplateCloneServiceTest {
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
@MockBean
|
||||
private TypeRepository typeRepository;
|
||||
@MockBean
|
||||
private ComponentMappingService componentMappingService;
|
||||
|
||||
private DossierTemplateCloneService dossierTemplateCloneService;
|
||||
|
||||
@ -124,7 +127,8 @@ public class DossierTemplateCloneServiceTest {
|
||||
storageService,
|
||||
dossierStatusPersistenceService,
|
||||
watermarkService,
|
||||
fileManagementStorageService);
|
||||
fileManagementStorageService,
|
||||
componentMappingService);
|
||||
|
||||
dummyTemplate = new DossierTemplateEntity();
|
||||
setNonDefaultValues(dummyTemplate);
|
||||
@ -169,7 +173,7 @@ public class DossierTemplateCloneServiceTest {
|
||||
|
||||
if (getterMethod != null) {
|
||||
value = getterMethod.invoke(obj);
|
||||
} else if (alternativeGetterMethod != null){
|
||||
} else if (alternativeGetterMethod != null) {
|
||||
value = alternativeGetterMethod.invoke(obj);
|
||||
} else {
|
||||
return;
|
||||
|
||||
@ -0,0 +1,173 @@
|
||||
package com.iqser.red.service.peristence.v1.server.integration.tests;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.util.FileSystemUtils;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.export.DossierTemplateExportService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
public class DossierTemplateImportExportTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
public static final String IMPORTED_TEMPLATE_NAME = "imported-template";
|
||||
public static final String USER_ID = "NutzerIdentifikationsNummer";
|
||||
public static final String AFTER = "after";
|
||||
public static final String BEFORE = "before";
|
||||
|
||||
@Autowired
|
||||
DossierTemplateImportService dossierTemplateImportService;
|
||||
|
||||
@Autowired
|
||||
DossierTemplateExportService dossierTemplateExportService;
|
||||
|
||||
@Autowired
|
||||
StorageService storageService;
|
||||
|
||||
@MockBean
|
||||
DownloadStatusPersistenceService downloadStatusPersistenceService;
|
||||
|
||||
|
||||
@AfterEach
|
||||
public void clearStorage() {
|
||||
|
||||
((FileSystemBackedStorageService) storageService).clearStorage();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
// @DirtiesContext
|
||||
public void testImportExportRoundtrip() {
|
||||
|
||||
TenantContext.setTenantId(TENANT_1);
|
||||
Path outDir = Files.createTempDirectory(IMPORTED_TEMPLATE_NAME);
|
||||
|
||||
Path dossierTemplateExportArchive = new ClassPathResource("files/dossiertemplates/DossierTemplate.zip").getFile().toPath();
|
||||
|
||||
String importedDossierTemplateId = dossierTemplateImportService.importDossierTemplate(ImportDossierTemplateRequest.builder()
|
||||
.archive(Files.readAllBytes(dossierTemplateExportArchive))
|
||||
.userId(USER_ID)
|
||||
.build());
|
||||
|
||||
when(downloadStatusPersistenceService.getStatus(anyString()))//
|
||||
.thenReturn(DownloadStatusEntity.builder().filename(importedDossierTemplateId + ".zip").storageId(IMPORTED_TEMPLATE_NAME).build());
|
||||
|
||||
dossierTemplateExportService.createDownloadArchive(DownloadJob.builder().storageId(importedDossierTemplateId).userId(USER_ID).build());
|
||||
|
||||
File tmpFile = Files.createTempFile(IMPORTED_TEMPLATE_NAME, ".zip").toFile();
|
||||
storageService.downloadTo(TenantContext.getTenantId(), IMPORTED_TEMPLATE_NAME, tmpFile);
|
||||
|
||||
unzip(dossierTemplateExportArchive.toFile().toString(), outDir.resolve(BEFORE));
|
||||
unzip(tmpFile.toString(), outDir.resolve(AFTER));
|
||||
|
||||
assert tmpFile.delete();
|
||||
|
||||
Map<Path, byte[]> beforeContents = getDirectoryContents(outDir.resolve(BEFORE));
|
||||
Map<Path, byte[]> afterContents = getDirectoryContents(outDir.resolve(AFTER));
|
||||
|
||||
assertEquals(beforeContents.size(), afterContents.size());
|
||||
assertEquals(beforeContents.keySet(), afterContents.keySet());
|
||||
|
||||
// can't assert equality on contents as UUID's are supposed to change on import
|
||||
|
||||
FileSystemUtils.deleteRecursively(outDir);
|
||||
}
|
||||
|
||||
|
||||
public static void unzip(String zipFilePath, Path destDirectory) throws IOException {
|
||||
|
||||
File destDir = destDirectory.toFile();
|
||||
if (!destDir.exists()) {
|
||||
destDir.mkdir();
|
||||
}
|
||||
try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath))) {
|
||||
ZipEntry entry = zipIn.getNextEntry();
|
||||
while (entry != null) {
|
||||
String filePath = destDirectory + File.separator + entry.getName();
|
||||
if (!entry.isDirectory()) {
|
||||
// if the entry is a file, extracts it
|
||||
extractFile(zipIn, filePath);
|
||||
} else {
|
||||
// if the entry is a directory, make the directory
|
||||
File dir = new File(filePath);
|
||||
dir.mkdirs();
|
||||
}
|
||||
zipIn.closeEntry();
|
||||
entry = zipIn.getNextEntry();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static void extractFile(ZipInputStream zipIn, String filePath) throws IOException {
|
||||
|
||||
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) {
|
||||
byte[] bytesIn = new byte[4096];
|
||||
int read;
|
||||
while ((read = zipIn.read(bytesIn)) != -1) {
|
||||
bos.write(bytesIn, 0, read);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static Map<Path, byte[]> getDirectoryContents(Path dir) throws IOException, NoSuchAlgorithmException {
|
||||
|
||||
Map<Path, byte[]> contents = new HashMap<>();
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
|
||||
Files.walkFileTree(dir, new SimpleFileVisitor<>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
|
||||
|
||||
contents.put(dir.relativize(file), digest.digest(Files.readAllBytes(file)));
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public FileVisitResult preVisitDirectory(Path subDir, BasicFileAttributes attrs) {
|
||||
|
||||
contents.put(dir.relativize(subDir), new byte[0]);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
|
||||
return contents;
|
||||
}
|
||||
|
||||
}
|
||||
@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
@ -298,7 +299,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
|
||||
var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k);
|
||||
|
||||
if (k == 1) {
|
||||
fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1);
|
||||
fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1, Collections.emptyList());
|
||||
reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k))));
|
||||
}
|
||||
if (k == 2) {
|
||||
|
||||
@ -585,7 +585,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
var status = statuses.getDownloadStatus()
|
||||
.iterator().next();
|
||||
exportDownloadReportMessageReceiver.receive(new DownloadJob(status.getUserId(), status.getStorageId(), false));
|
||||
exportDownloadReportMessageReceiver.receive(new DownloadJob(status.getUserId(), status.getStorageId()));
|
||||
|
||||
// add new justifications
|
||||
legalBasisClient.setLegalBasisMapping(List.of(new LegalBasis("nameAgain", "description", "reason")), dossierTemplate.getId());
|
||||
|
||||
@ -99,7 +99,7 @@ public class DownloadTest extends AbstractPersistenceServerServiceTest {
|
||||
.fileIds(List.of(file2.getId()))
|
||||
.build());
|
||||
|
||||
downloadMessageReceiver.receive(new DownloadJob(userProvider.getUserId(), downloads.getStorageId(), false));
|
||||
downloadMessageReceiver.receive(new DownloadJob(userProvider.getUserId(), downloads.getStorageId()));
|
||||
var reportInfoId = downloads.getStorageId().substring(0, downloads.getStorageId().length() - 3) + "/REPORT_INFO.json";
|
||||
storageService.storeJSONObject(TenantContext.getTenantId(), reportInfoId, new ArrayList<>());
|
||||
|
||||
|
||||
@ -9,14 +9,17 @@ import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository;
|
||||
|
||||
import org.assertj.core.util.Lists;
|
||||
import org.bson.BsonArray;
|
||||
import org.bson.BsonDocument;
|
||||
import org.bson.BsonString;
|
||||
import org.checkerframework.checker.units.qual.A;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@ -38,6 +41,7 @@ import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.core.env.ConfigurableEnvironment;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.StatementCallback;
|
||||
import org.springframework.jdbc.datasource.SingleConnectionDataSource;
|
||||
@ -50,8 +54,10 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
|
||||
import org.springframework.security.crypto.password.PasswordEncoder;
|
||||
import org.springframework.security.provisioning.InMemoryUserDetailsManager;
|
||||
import org.springframework.security.web.SecurityFilterChain;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
import org.springframework.test.context.support.TestPropertySourceUtils;
|
||||
|
||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||
import com.iqser.red.service.peristence.v1.server.Application;
|
||||
@ -138,6 +144,9 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = "spring-hibernate-query-utils.n-plus-one-queries-detection.error-level=INFO")
|
||||
public abstract class AbstractPersistenceServerServiceTest {
|
||||
|
||||
public static final String TENANT_1 = "redaction";
|
||||
public static final String TENANT_2 = "redaction2";
|
||||
|
||||
@MockBean
|
||||
protected Scheduler scheduler;
|
||||
@MockBean
|
||||
@ -273,7 +282,7 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
@BeforeEach
|
||||
public void setupTenantContext() {
|
||||
|
||||
TenantContext.setTenantId("redaction");
|
||||
TenantContext.setTenantId(TENANT_1);
|
||||
}
|
||||
|
||||
|
||||
@ -289,7 +298,7 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
|
||||
createTenants();
|
||||
|
||||
TenantContext.setTenantId("redaction");
|
||||
TenantContext.setTenantId(TENANT_1);
|
||||
|
||||
ApplicationConfig appConfig = ApplicationConfig.builder().downloadCleanupDownloadFilesHours(8).downloadCleanupNotDownloadFilesHours(72).softDeleteCleanupTime(96).build();
|
||||
applicationConfigService.saveApplicationConfiguration(MagicConverter.convert(appConfig, ApplicationConfigurationEntity.class));
|
||||
@ -346,9 +355,9 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
if (tenantsClient.getTenants() == null || tenantsClient.getTenants().isEmpty()) {
|
||||
|
||||
var redactionTenant = new TenantResponse();
|
||||
redactionTenant.setTenantId("redaction");
|
||||
redactionTenant.setGuid("redaction");
|
||||
redactionTenant.setDisplayName("redaction");
|
||||
redactionTenant.setTenantId(TENANT_1);
|
||||
redactionTenant.setGuid(TENANT_1);
|
||||
redactionTenant.setDisplayName(TENANT_1);
|
||||
redactionTenant.setAuthDetails(new AuthDetails());
|
||||
redactionTenant.setDatabaseConnection(DatabaseConnection.builder()
|
||||
.driver("postgresql")
|
||||
@ -388,9 +397,9 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
.build());
|
||||
|
||||
var redactionTenant2 = new TenantResponse();
|
||||
redactionTenant2.setTenantId("redaction2");
|
||||
redactionTenant2.setGuid("redaction2");
|
||||
redactionTenant2.setDisplayName("redaction2");
|
||||
redactionTenant2.setTenantId(TENANT_2);
|
||||
redactionTenant2.setGuid(TENANT_2);
|
||||
redactionTenant2.setDisplayName(TENANT_2);
|
||||
redactionTenant2.setAuthDetails(new AuthDetails());
|
||||
redactionTenant2.setDatabaseConnection(DatabaseConnection.builder()
|
||||
.driver("postgresql")
|
||||
@ -425,19 +434,19 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
.username(MONGO_USERNAME)
|
||||
.password(encryptionDecryptionService.encrypt(MONGO_PASSWORD))
|
||||
.address(mongoDbContainer.getHost() + ":" + mongoDbContainer.getFirstMappedPort())
|
||||
.database("redaction2")
|
||||
.database(TENANT_2)
|
||||
.options("")
|
||||
.build());
|
||||
|
||||
when(tenantsClient.getTenant("redaction")).thenReturn(redactionTenant);
|
||||
when(tenantsClient.getTenant("redaction2")).thenReturn(redactionTenant2);
|
||||
when(tenantsClient.getTenant(TENANT_1)).thenReturn(redactionTenant);
|
||||
when(tenantsClient.getTenant(TENANT_2)).thenReturn(redactionTenant2);
|
||||
when(tenantsClient.getTenants()).thenReturn(List.of(redactionTenant, redactionTenant2));
|
||||
|
||||
try {
|
||||
tenantCreatedListener.createTenant(new TenantCreatedEvent("redaction"));
|
||||
tenantCreatedListener.createTenant(new TenantCreatedEvent("redaction2"));
|
||||
mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent("redaction"));
|
||||
mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent("redaction2"));
|
||||
tenantCreatedListener.createTenant(new TenantCreatedEvent(TENANT_1));
|
||||
tenantCreatedListener.createTenant(new TenantCreatedEvent(TENANT_2));
|
||||
mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent(TENANT_1));
|
||||
mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent(TENANT_2));
|
||||
} catch (Exception e) {
|
||||
|
||||
e.printStackTrace();
|
||||
@ -515,7 +524,6 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
notificationPreferencesRepository.deleteAll();
|
||||
indexInformationRepository.deleteAll();
|
||||
applicationConfigRepository.deleteAll();
|
||||
|
||||
entityLogEntryDocumentRepository.deleteAll();
|
||||
entityLogDocumentRepository.deleteAll();
|
||||
|
||||
@ -526,8 +534,16 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
@Slf4j
|
||||
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
|
||||
|
||||
static AtomicInteger UniquePortFactory = new AtomicInteger(28081);
|
||||
|
||||
|
||||
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
|
||||
|
||||
// any time the WebEnvironment is re-instantiated use a new port, as it sometimes leads to PortBindExceptions
|
||||
ConfigurableEnvironment environment = configurableApplicationContext.getEnvironment();
|
||||
int port = UniquePortFactory.getAndIncrement();
|
||||
TestPropertySourceUtils.addInlinedPropertiesToEnvironment(environment, "server.port=" + port);
|
||||
|
||||
var postgreSQLContainerMaster = SpringPostgreSQLTestContainer.getInstance().withDatabaseName("integration-tests-db-master").withUsername("sa").withPassword("sa");
|
||||
|
||||
postgreSQLContainerMaster.start();
|
||||
@ -537,8 +553,8 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
|
||||
var mongoInstance = MongoDBTestContainer.getInstance();
|
||||
mongoInstance.start();
|
||||
createMongoDBDatabase(mongoInstance, "redaction");
|
||||
createMongoDBDatabase(mongoInstance, "redaction2");
|
||||
createMongoDBDatabase(mongoInstance, TENANT_1);
|
||||
createMongoDBDatabase(mongoInstance, TENANT_2);
|
||||
|
||||
log.info("Hosts are - Redis: {}, Postgres: {}, MongoDB: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost(), mongoInstance.getHost());
|
||||
|
||||
@ -553,37 +569,36 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer, String databaseName) {
|
||||
|
||||
private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer, String databaseName) {
|
||||
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
|
||||
MONGO_USERNAME,
|
||||
MONGO_PASSWORD,
|
||||
mongoDBTestContainer.getHost(),
|
||||
mongoDBTestContainer.getFirstMappedPort()))) {
|
||||
MongoDatabase database = mongoClient.getDatabase(databaseName);
|
||||
BsonDocument createUserCommand = new BsonDocument();
|
||||
createUserCommand.append("createUser", new BsonString(MONGO_USERNAME));
|
||||
createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD));
|
||||
BsonArray roles = new BsonArray();
|
||||
roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(databaseName)));
|
||||
createUserCommand.append("roles", roles);
|
||||
|
||||
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
|
||||
MONGO_USERNAME,
|
||||
MONGO_PASSWORD,
|
||||
mongoDBTestContainer.getHost(),
|
||||
mongoDBTestContainer.getFirstMappedPort()))) {
|
||||
MongoDatabase database = mongoClient.getDatabase(databaseName);
|
||||
BsonDocument createUserCommand = new BsonDocument();
|
||||
createUserCommand.append("createUser", new BsonString(MONGO_USERNAME));
|
||||
createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD));
|
||||
BsonArray roles = new BsonArray();
|
||||
roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(databaseName)));
|
||||
createUserCommand.append("roles", roles);
|
||||
try {
|
||||
database.runCommand(createUserCommand);
|
||||
} catch (MongoCommandException mongoCommandException) {
|
||||
// ignore user already exists
|
||||
if (mongoCommandException.getErrorCode() != 51003) {
|
||||
throw mongoCommandException;
|
||||
}
|
||||
|
||||
try {
|
||||
database.runCommand(createUserCommand);
|
||||
} catch (MongoCommandException mongoCommandException) {
|
||||
// ignore user already exists
|
||||
if (mongoCommandException.getErrorCode() != 51003) {
|
||||
throw mongoCommandException;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Configuration
|
||||
@EnableWebSecurity
|
||||
|
||||
Binary file not shown.
@ -7,6 +7,7 @@ import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
@ -34,5 +35,8 @@ public class AnalyzeRequest {
|
||||
@Builder.Default
|
||||
private List<FileAttribute> fileAttributes = new ArrayList<>();
|
||||
|
||||
@Builder.Default
|
||||
private List<ComponentMappingMetadata> componentMappings = new ArrayList<>();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
package com.iqser.red.service.persistence.service.v1.api.shared.model;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
@ -37,4 +39,6 @@ public class AnalyzeResult {
|
||||
|
||||
private Set<FileAttribute> addedFileAttributes;
|
||||
|
||||
private List<ComponentMappingMetadata> usedComponentMappings;
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package com.iqser.red.service.persistence.service.v1.api.shared.model;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
@ -147,7 +148,8 @@ public class FileStatus {
|
||||
private OffsetDateTime lastIndexed;
|
||||
@Schema(description = "The error information for the error state of the file")
|
||||
private FileErrorInfo fileErrorInfo;
|
||||
|
||||
@Schema(description = "Shows which versions of each mapping the last analysis has been performed")
|
||||
private Map<String, Integer> componentMappingVersions;
|
||||
|
||||
@Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.")
|
||||
public OffsetDateTime getLastOCRTime() {
|
||||
|
||||
@ -0,0 +1,38 @@
|
||||
package com.iqser.red.service.persistence.service.v1.api.shared.model.component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.NonNull;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class ComponentMappingMetadata {
|
||||
|
||||
String id;
|
||||
|
||||
String name;
|
||||
|
||||
String fileName;
|
||||
|
||||
String storageId;
|
||||
|
||||
Integer version;
|
||||
|
||||
List<String> columnLabels;
|
||||
|
||||
Integer numberOfLines;
|
||||
|
||||
String encoding;
|
||||
|
||||
char delimiter;
|
||||
|
||||
}
|
||||
@ -73,6 +73,7 @@ public class FileModel {
|
||||
private OffsetDateTime fileManipulationDate;
|
||||
private boolean hasHighlights;
|
||||
private FileErrorInfo fileErrorInfo;
|
||||
private Map<String, Integer> componentMappingVersions = new HashMap<>();
|
||||
|
||||
|
||||
public long getFileSize() {
|
||||
|
||||
@ -19,6 +19,5 @@ public enum ProcessingStatus {
|
||||
PRE_PROCESSED,
|
||||
FIGURE_DETECTION_ANALYZING,
|
||||
TABLE_PARSING_ANALYZING,
|
||||
|
||||
VISUAL_LAYOUT_PARSING_ANALYZING
|
||||
}
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||
|
||||
public record ComponentMappingImportModel(ComponentMappingMetadata metadata, byte[] csvData) {
|
||||
|
||||
}
|
||||
@ -19,7 +19,9 @@ public enum ExportFilename {
|
||||
DOSSIER_TYPE("dossierType"),
|
||||
ENTRIES("entries"),
|
||||
FALSE_POSITIVES("falsePositives"),
|
||||
FALSE_RECOMMENDATION("falseRecommendations");
|
||||
FALSE_RECOMMENDATION("falseRecommendations"),
|
||||
COMPONENT_MAPPINGS_FOLDER("mappings"),
|
||||
COMPONENT_MAPPINGS_FILE("componentMappingsList");
|
||||
|
||||
@Getter
|
||||
private final String filename;
|
||||
|
||||
@ -50,6 +50,9 @@ public class ImportTemplateResult {
|
||||
@Builder.Default
|
||||
public List<FileAttributeConfig> fileAttributesConfigs = new ArrayList<>();
|
||||
|
||||
@Builder.Default
|
||||
public List<ComponentMappingImportModel> componentMappings = new ArrayList<>();
|
||||
|
||||
@Builder.Default
|
||||
public List<LegalBasis> legalBases = new ArrayList<>();
|
||||
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
dir=${PWD##*/}
|
||||
|
||||
gradle assemble
|
||||
|
||||
# Get the current Git branch
|
||||
@ -11,5 +15,31 @@ commit_hash=$(git rev-parse --short=5 HEAD)
|
||||
# Combine branch and commit hash
|
||||
buildName="${USER}-${branch}-${commit_hash}"
|
||||
|
||||
gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
|
||||
echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"
|
||||
gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${buildName}
|
||||
|
||||
newImageName="nexus.knecon.com:5001/red/${dir}-server-v1:${buildName}"
|
||||
|
||||
echo "full image name:"
|
||||
echo ${newImageName}
|
||||
echo ""
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
namespace=${1}
|
||||
deployment_name="persistence-service-v1"
|
||||
|
||||
echo "deploying to ${namespace}"
|
||||
|
||||
oldImageName=$(rancher kubectl -n ${namespace} get deployment ${deployment_name} -o=jsonpath='{.spec.template.spec.containers[*].image}')
|
||||
|
||||
if [ "${newImageName}" = "${oldImageName}" ]; then
|
||||
echo "Image tag did not change, redeploying..."
|
||||
rancher kubectl rollout restart deployment ${deployment_name} -n ${namespace}
|
||||
else
|
||||
echo "upgrading the image tag..."
|
||||
rancher kubectl set image deployment/${deployment_name} ${deployment_name}=${newImageName} -n ${namespace}
|
||||
fi
|
||||
rancher kubectl rollout status deployment ${deployment_name} -n ${namespace}
|
||||
echo "Built ${deployment_name}:${buildName} and deployed to ${namespace}"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user