diff --git a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts index c6b715907..eeff524f4 100644 --- a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts @@ -65,6 +65,15 @@ java { } allprojects { + + tasks.withType { + options { + this as StandardJavadocDocletOptions + addBooleanOption("Xdoclint:none", true) + addStringOption("Xmaxwarns", "1") + } + } + publishing { publications { create(name) { diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts b/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts index 88086af39..fc349e6ef 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/build.gradle.kts @@ -7,6 +7,9 @@ dependencies { api(project(":persistence-service-processor-v1")) api(project(":persistence-service-external-api-v2")) api(project(":persistence-service-external-api-impl-v1")) + + implementation("org.mapstruct:mapstruct:1.5.5.Final") + annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") } description = "persistence-service-external-api-impl-v2" diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java index 097de9abb..943fba2d0 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java @@ -12,10 +12,10 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierTemplateController; import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController; import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntityReference; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue; @@ -37,10 +37,10 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ComponentControllerV2 implements ComponentResource { - DossierTemplateController dossierTemplateController; ComponentLogService componentLogService; StatusController statusController; FileStatusService fileStatusService; + DossierTemplatePersistenceService dossierTemplatePersistenceService; @Override @@ -49,7 +49,7 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(FILE_ID_PARAM) String fileId, @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) { - dossierTemplateController.getDossierTemplate(dossierTemplateId); + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); var componentLog = componentLogService.getComponentLog(dossierId, fileId, true); Map> basicComponent = new LinkedHashMap<>(); @@ -119,11 +119,12 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) { - dossierTemplateController.getDossierTemplate(dossierTemplateId); + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); var dossierFiles = statusController.getDossierStatus(dossierId); return new FileComponentsList(dossierFiles.stream() .map(file -> getComponents(dossierTemplateId, dossierId, file.getFileId(), includeDetails)) .toList()); } + } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java index 2787e550f..3662294a1 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java @@ -6,8 +6,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.WRITE_RULES; import java.io.ByteArrayInputStream; +import java.io.FileOutputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Locale; @@ -23,11 +26,17 @@ import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import com.google.common.base.Strings; import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierTemplateController; import com.iqser.red.persistence.service.v1.external.api.impl.controller.FileAttributesController; +import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMappingMapper; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.utils.RulesValidationMapper; import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils; @@ -37,6 +46,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinition; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList; import com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource; @@ -46,21 +57,27 @@ import feign.FeignException; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; @RestController @RequiredArgsConstructor @Tag(name = "1. Dossier templates endpoints", description = "Provides operations related to dossier templates") +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DossierTemplateControllerV2 implements DossierTemplateResource { private static final String RULES_DOWNLOAD_FILE_NAME_SUFFIX = "-rules.drl"; - private final DossierTemplateController dossierTemplateController; - private final RulesPersistenceService rulesPersistenceService; - private final RulesValidationService rulesValidationService; - private final AuditPersistenceService auditPersistenceService; - private final FileAttributesController fileAttributesController; + DossierTemplateController dossierTemplateController; + RulesPersistenceService rulesPersistenceService; + RulesValidationService rulesValidationService; + AuditPersistenceService auditPersistenceService; + FileAttributesController fileAttributesController; + ComponentMappingService componentMappingService; + ComponentMappingMapper componentMappingMapper = ComponentMappingMapper.INSTANCE; + DossierTemplatePersistenceService dossierTemplatePersistenceService; public List getAllDossierTemplates() { @@ -162,6 +179,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { .dossierTemplateId(dossierTemplateId) .ruleFileType(ruleFileType) .build(); + DroolsValidationResponse rulesValidationResponse = new DroolsValidationResponse(); try { @@ -194,7 +212,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { private ResponseEntity downloadRules(String dossierTemplateId, RuleFileType ruleFileType) { - var ruleEntity = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType); + RuleSetEntity ruleEntity = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType); var data = ruleEntity.getValue().getBytes(StandardCharsets.UTF_8); @@ -209,4 +227,112 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { return new ResponseEntity<>(new InputStreamResource(is), httpHeaders, HttpStatus.OK); } + + @Override + @PreAuthorize("hasAuthority('" + READ_RULES + "')") + public ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + List summaries = componentMappingService.getMetaDataByDossierTemplateId( + dossierTemplateId); + List componentMappingMetadataModelList = componentMappingMapper.toModelList(summaries); + return new ComponentMappingSummary(dossierTemplateId, componentMappingMetadataModelList); + } + + + @Override + @SneakyThrows + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, char delimiter) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + String nameToUse = Strings.isNullOrEmpty(name) ? file.getName().split("\\.")[0] : name; + + if (Strings.isNullOrEmpty(nameToUse)) { + throw new BadRequestException("The provided file name is not valid!"); + } + + Path mappingFile = saveToFile(file); + String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); + + com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, + nameToUse, + fileName, + delimiter, + encoding, + mappingFile.toFile()); + + Files.deleteIfExists(mappingFile); + + return componentMappingMapper.toModel(metaData); + } + + + @Override + @SneakyThrows + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String encoding, char delimiter) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + Path mappingFile = saveToFile(file); + + com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId, + componentMappingId, + encoding, + delimiter, + mappingFile.toFile()); + + Files.deleteIfExists(mappingFile); + + return componentMappingMapper.toModel(resultMetaData); + } + + + @Override + @PreAuthorize("hasAuthority('" + READ_RULES + "')") + public ResponseEntity downloadMapping(String dossierTemplateId, String componentMappingId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + ComponentMappingDownloadModel mappingDownloadModel = componentMappingService.getMappingForDownload(dossierTemplateId, componentMappingId); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.TEXT_PLAIN); + + httpHeaders.add("Content-Disposition", + "attachment" + + "; filename*=" + + mappingDownloadModel.encoding().toLowerCase(Locale.US) + + "''" + + StringEncodingUtils.urlEncode(mappingDownloadModel.fileName())); + + return new ResponseEntity<>(mappingDownloadModel.mappingFileResource(), httpHeaders, HttpStatus.OK); + } + + + @Override + @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") + public ResponseEntity deleteMapping(String dossierTemplateId, String componentMappingId) { + + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + + componentMappingService.delete(dossierTemplateId, componentMappingId); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + + @SneakyThrows + private static Path saveToFile(MultipartFile file) { + + Path mappingFile = Files.createTempFile(file.getName(), ".csv"); + + try (var out = new FileOutputStream(mappingFile.toFile())) { + out.write(file.getBytes()); + } + return mappingFile; + } + } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java new file mode 100644 index 000000000..b22db8eb7 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java @@ -0,0 +1,184 @@ +package com.iqser.red.persistence.service.v2.external.api.impl.controller; + +import java.time.OffsetDateTime; +import java.util.Map; +import java.util.stream.Collectors; + +import org.quartz.JobDataMap; +import org.quartz.JobKey; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.HttpStatus; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.WebDataBinder; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.multipart.support.MissingServletRequestPartException; + +import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import com.iqser.red.commons.spring.ErrorMessage; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.knecon.fforesight.tenantcommons.TenantContext; +import com.mchange.rmi.NotAuthorizedException; + +import io.swagger.v3.oas.annotations.Hidden; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RestControllerAdvice +@RequiredArgsConstructor +@Order(Ordered.HIGHEST_PRECEDENCE) +public class ExternalControllerAdviceV2 { + + private final Scheduler scheduler; + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.NOT_FOUND) + @ExceptionHandler(value = NotFoundException.class) + public ErrorMessage handleContentNotFoundException(NotFoundException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + /* error handling */ + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler(value = BadRequestException.class) + public ErrorMessage handleBadRequestException(BadRequestException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.CONFLICT) + @ExceptionHandler(value = {ConflictException.class}) + protected ErrorMessage handleConflictException(ConflictException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.FORBIDDEN) + @ExceptionHandler({AccessDeniedException.class}) + public ErrorMessage handleAccessDeniedException(AccessDeniedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.UNAUTHORIZED) + @ExceptionHandler({NotAuthorizedException.class}) + public ErrorMessage handleNotAuthorizedException(NotAuthorizedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @ResponseBody + @ResponseStatus(value = HttpStatus.FORBIDDEN) + @ExceptionHandler({NotAllowedException.class}) + public ErrorMessage handleNotAllowedException(NotAllowedException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + @ExceptionHandler({org.springframework.security.acls.model.NotFoundException.class}) + public ErrorMessage handleACLNotFound(org.springframework.security.acls.model.NotFoundException e) { + + // in case this error occurs on a rest request / force trigger the sync job + try { + scheduler.triggerJob(new JobKey("SyncUserPermissionsJob"), new JobDataMap(Map.of("tenantId", TenantContext.getTenantId()))); + } catch (SchedulerException ex) { + log.debug("Failed to force trigger SyncUserPermissionsJob", ex); + } + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler({MethodArgumentNotValidException.class}) + public ErrorMessage handleMethodArgumentNotValidException(MethodArgumentNotValidException e) { + + var errorList = e.getFieldErrors(); + String errorListAsString = errorList.stream() + .map(fieldError -> fieldError.getField() + ": " + fieldError.getDefaultMessage()) + .collect(Collectors.joining(", ")); + return new ErrorMessage(OffsetDateTime.now(), String.format("You have empty/wrong formatted parameters: %s", errorListAsString)); + } + + + @Hidden + @ResponseBody + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + @ExceptionHandler({MissingServletRequestPartException.class}) + public ErrorMessage handleMissingServletRequestPartException(MissingServletRequestPartException e) { + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Hidden + @ResponseBody + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler({HttpMessageNotReadableException.class}) + public ErrorMessage handleHttpMessageNotReadableException(HttpMessageNotReadableException e) { + + var cause = e.getCause(); + if (cause instanceof InvalidFormatException) { + InvalidFormatException invalidFormatException = (InvalidFormatException) cause; + + Class targetType = invalidFormatException.getTargetType(); + if (targetType != null && targetType.isEnum()) { + return new ErrorMessage(OffsetDateTime.now(), String.format("Unsupported value for %s", targetType.getSimpleName())); + } + + return new ErrorMessage(OffsetDateTime.now(), cause.getMessage()); + } + + return new ErrorMessage(OffsetDateTime.now(), e.getMessage()); + } + + + @Order(10000) + public static class BinderControllerAdvice { + + @InitBinder + public void setAllowedFields(WebDataBinder dataBinder) { + // This code protects Spring Core from a "Remote Code Execution" attack (dubbed "Spring4Shell"). + // By applying this mitigation, you prevent the "Class Loader Manipulation" attack vector from firing. + // For more details, see this post: https://www.lunasec.io/docs/blog/spring-rce-vulnerabilities/ + String[] denylist = new String[]{"class.*", "Class.*", "*.class.*", "*.Class.*"}; + dataBinder.setDisallowedFields(denylist); + } + + } + +} diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMappingMapper.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMappingMapper.java new file mode 100644 index 000000000..975db2bf8 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMappingMapper.java @@ -0,0 +1,28 @@ +package com.iqser.red.persistence.service.v2.external.api.impl.mapper; + +import java.util.List; + +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; + +@Mapper +public interface ComponentMappingMapper { + + ComponentMappingMapper INSTANCE = Mappers.getMapper(ComponentMappingMapper.class); + + + ComponentMappingMetadataModel toModel(ComponentMappingMetadata componentMappingMetadata); + + + List toModelList(List componentMappingMetadata); + + + ComponentMappingMetadata toDto(ComponentMappingMetadataModel componentMappingMetadataModel); + + + List toDtoList(List componentMappingMetadataModels); + +} diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingMetadataModel.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingMetadataModel.java new file mode 100644 index 000000000..b8da5546b --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingMetadataModel.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.persistence.service.v2.api.external.model; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ComponentMappingMetadataModel { + + String id; + String name; + String fileName; + Integer version; + List columnLabels; + Integer numberOfLines; + String encoding; + char delimiter; + +} diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingSummary.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingSummary.java new file mode 100644 index 000000000..7a807225a --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentMappingSummary.java @@ -0,0 +1,22 @@ +package com.iqser.red.service.persistence.service.v2.api.external.model; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ComponentMappingSummary { + + String dossierTemplateId; + List componentMappingMetadata; + +} diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java index ca0cdbec1..a67b78e9c 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java @@ -8,20 +8,29 @@ import static com.iqser.red.service.persistence.service.v2.api.external.resource import static com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource.DOSSIER_TEMPLATE_PATH; import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_ID_PARAM; import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_ID_PATH_VARIABLE; -import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_PATH; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.multipart.MultipartFile; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents; import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponentsList; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -29,7 +38,8 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; @ResponseStatus(value = HttpStatus.OK) public interface ComponentResource { - String PATH = ExternalApiConstants.BASE_PATH + DOSSIER_TEMPLATE_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + DOSSIER_PATH + DOSSIER_ID_PATH_PARAM + FILE_PATH; + String PATH = ExternalApiConstants.BASE_PATH + DOSSIER_TEMPLATE_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE; + String FILE_PATH = PATH + DOSSIER_PATH + DOSSIER_ID_PATH_PARAM + FileResource.FILE_PATH; String COMPONENTS_PATH = "/components"; @@ -39,13 +49,13 @@ public interface ComponentResource { String INCLUDE_DETAILS_DESCRIPTION = """ A toggle to decide whether to include detailed component information in the response: - - true: The component object's field componentDetails stores detailed information about the source of its respective value(s). - false (default): The component object does not contain a field componentDetails. """; - @GetMapping(value = PATH + FILE_ID_PATH_VARIABLE + COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) + + @GetMapping(value = FILE_PATH + FILE_ID_PATH_VARIABLE + COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) @Operation(summary = "Returns the components for a file", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) FileComponents getComponents(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @@ -54,11 +64,13 @@ public interface ComponentResource { @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails); - @GetMapping(value = PATH + BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) + @GetMapping(value = FILE_PATH + BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) @Operation(summary = "Returns the components for all files of a dossier", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) FileComponentsList getComponentsOfDossier(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier that contains the file.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId, @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails); + + } diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java index 78c1ec44e..7257cdd41 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java @@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.service.v2.api.external.resource; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingSummary; import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList; import io.swagger.v3.oas.annotations.Operation; @@ -13,9 +15,11 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.ResponseBody; @@ -33,15 +37,20 @@ public interface DossierTemplateResource { String ENTITY_RULES_PATH = "/entity-rules"; String COMPONENT_RULES_PATH = "/component-rules"; + String COMPONENT_MAPPINGS_PATH = "/component-mappings"; String FILE_ATTRIBUTE_DEFINITIONS_PATH = "/file-attribute-definitions"; String DOSSIER_TEMPLATE_ID_PARAM = "dossierTemplateId"; String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID_PARAM + "}"; - String RULE_FILE_TYPE_PARAMETER_NAME = "ruleFileType"; + + String COMPONENT_MAPPING_ID_PARAM = "componentMappingId"; + String COMPONENT_MAPPING_ID_PATH_VARIABLE = "/{" + COMPONENT_MAPPING_ID_PARAM + "}"; String DRY_RUN_PARAM = "dryRun"; - + String ENCODING_PARAM = "encoding"; + String DELIMITER_PARAM = "delimiter"; + String MAPPING_NAME_PARAM = "name"; @GetMapping(value = PATH, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Lists all existing DossierTemplates.", description = "None") @@ -60,8 +69,8 @@ public interface DossierTemplateResource { @Operation(summary = "Upload a component or entity rules file in drools format for a specific DossierTemplate.") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Rules upload successful."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found."), @ApiResponse(responseCode = "400", description = "Uploaded rules could not be verified."), @ApiResponse(responseCode = "422", description = "Uploaded rules could not be compiled.")}) ResponseEntity uploadEntityRules(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, - @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, - @Parameter(name = DRY_RUN_PARAM, description = "If true rules will be only validated not stored.") @RequestParam(value = DRY_RUN_PARAM, required = false, defaultValue = "false") boolean dryRun); + @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @Parameter(name = DRY_RUN_PARAM, description = "If true rules will be only validated not stored.") @RequestParam(value = DRY_RUN_PARAM, required = false, defaultValue = "false") boolean dryRun); @ResponseBody @@ -96,4 +105,51 @@ public interface DossierTemplateResource { @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "File attribute definitions returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) FileAttributeDefinitionList getFileAttributeDefinitions(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId); + + @Operation(summary = "Get the component mapping summaries of a DossierTemplate.", description = "None") + @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping views returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")}) + ComponentMappingSummary getComponentMappingSummaries(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId); + + + @Operation(summary = "Upload a new component mapping to a DossierTemplate.", description = "None") + @PostMapping(value = PATH + + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + + COMPONENT_MAPPINGS_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping uploaded successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + ComponentMappingMetadataModel uploadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, + @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name, + @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, + @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); + + + @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None") + @PutMapping(value = PATH + + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + + COMPONENT_MAPPINGS_PATH + + COMPONENT_MAPPING_ID_PATH_VARIABLE, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Component mapping updated successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + ComponentMappingMetadataModel updateMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, + @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId, + @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, + @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); + + + @ResponseBody + @ResponseStatus(value = HttpStatus.OK) + @Operation(summary = "Returns file containing the specified mapping as a file.") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE) + ResponseEntity downloadMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId); + + + @ResponseBody + @ResponseStatus(value = HttpStatus.OK) + @Operation(summary = "Deletes a specified mapping.") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate or the specified mapping is not found.")}) + @DeleteMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + COMPONENT_MAPPINGS_PATH + COMPONENT_MAPPING_ID_PATH_VARIABLE) + ResponseEntity deleteMapping(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId, @PathVariable(COMPONENT_MAPPING_ID_PARAM) String componentMappingId); + } diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml new file mode 100644 index 000000000..62e0f1818 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml @@ -0,0 +1,3919 @@ +openapi: 3.0.2 +info: + title: DocuMine API + version: "1.0.0" + description: | + The DocuMine API provides a comprehensive solution for managing resources such as dossiers and their associated files. + Users can also retrieve components of files that have been processed and extracted by the system. + + All endpoints are secured using OAuth2, with the "authorizationCode" being the general supported authorization flow. + Obtain a JWT token for authentication and send it in the 'Authorization' header with the format `Bearer {JWT_TOKEN}`. + + Please also note that the `authorizationUrl` and `tokenUrl` in this specification contain `{workspaceId}` placeholders that + must be replaced by your respective DocuMine workspace identifier. + + Example Headers: + ```properties + Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI... + ``` + license: + name: knecon License + contact: + name: knecon Service Desk + email: support@documine.ai + url: https://support.documine.ai +externalDocs: + description: Find out more about DocuMine + url: https://docs.documine.ai +servers: + - url: https://app.documine.ai + description: DocuMine Cloud Service +security: + - FF-OAUTH: [ ] +tags: + - name: 1. Dossier Templates + description: Operations related to dossier templates. + - name: 2. Dossiers + description: Operations for managing dossiers that are based on a dossier template. + - name: 3. Files + description: Operations for managing files within a dossier. + - name: 4. Components + description: Operations related to components of a file within a dossier. + - name: 5. Downloads + description: Operations related to download packages. + - name: 6. Users + description: Operations related to users. + - name: 7. License + description: Operations related to license information and usage metrics. +paths: + /api/dossier-templates: + get: + operationId: getAllDossierTemplates + tags: + - 1. Dossier Templates + summary: Retrieve a list of all available dossier templates. + description: | + This endpoint provides a list of all dossier templates stored in the system. + + It returns an object containing an array of dossier template objects under the key `dossierTemplates`. + Each template object contains an identifier, a name, a dossier template status, and other fields with + further details about the template. + + Use this endpoint to fetch all templates before performing actions on specific ones. + responses: + "200": + content: + '*/*': + schema: + $ref: '#/components/schemas/DossierTemplateList' + description: | + Successfully retrieved the list of dossier templates. + + The response could be an empty list if there are no dossier templates at all. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}: + get: + operationId: getDossierTemplate + tags: + - 1. Dossier Templates + summary: Retrieve a specific dossier template by its identifier. + description: | + Utilize this endpoint to retrieve comprehensive details about a designated dossier template. The response encompasses + various attributes such as the dossier template's identifier, name, status, among other pertinent details. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DossierTemplate' + description: | + Successfully retrieved the requested dossier template. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/entity-rules: + get: + operationId: downloadEntityRules + tags: + - 1. Dossier Templates + summary: Download the entity rules of a specific dossier template. + description: | + Utilize this endpoint to download the entity rules of a designated dossier template. The file is named 'entity-rules.drl' + and contains the set of rules to annotate the entities in an analyzed file. The content of this file is in the Drools Rule Language + (DRL) format. Please find more details about the DRL in the + [Drools Language Reference](https://docs.drools.org/8.44.0.Final/drools-docs/drools/language-reference/index.html). + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + headers: + Content-Disposition: + schema: + type: string + example: attachment; filename*=utf-8''entity-rules.drl + content: + text/plain; charset=utf-8: + schema: + type: string + description: | + Successfully downloaded the requested rules file. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + post: + operationId: uploadEntityRules + tags: + - 1. Dossier Templates + summary: Upload or validate a entity rules file for a specific dossier template. + description: | + Utilize this endpoint to upload the entity rules to a designated dossier template. With the 'dryRun' parameter, + you have the possibility to just validate the rules file without actually saving it in the dossier template. + + The uploaded rule file will be saved only if 'dryRun' is set to `false` and the response code is `200`. In this + case, the response object does not contain errors. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dryRun' + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/RuleValidation' + example: + errors: [ ] + description: | + Successfully uploaded or validated the entity rules file. The returned response object will not contain any errors. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "422": + $ref: '#/components/responses/422-rules' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/component-rules: + get: + operationId: downloadComponentRules + tags: + - 1. Dossier Templates + summary: Download the component rules of a specific dossier template. + description: | + Utilize this endpoint to download the component rules of a designated dossier template. The file is named 'component-rules.drl' + and contains the set of rules to build components based on entities of an analyzed file. The content of this file is in the Drools Rule Language + (DRL) format. Please find more details about the DRL in the + [Drools Language Reference](https://docs.drools.org/8.44.0.Final/drools-docs/drools/language-reference/index.html). + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + headers: + Content-Disposition: + schema: + type: string + example: attachment; filename*=utf-8''component-rules.drl + content: + text/plain; charset=utf-8: + schema: + type: string + description: | + Successfully downloaded the requested rules file. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + post: + operationId: uploadComponentRules + tags: + - 1. Dossier Templates + summary: Upload or validate a component rules file for a specific dossier template. + description: | + Utilize this endpoint to upload the component rules to a designated dossier template. With the 'dryRun' parameter, + you have the possibility to just validate the rules file without actually saving it in the dossier template. + + The uploaded rule file will be saved only if 'dryRun' is set to `false` and the response code is `200`. In this + case, the response object does not contain errors. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dryRun' + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/RuleValidation' + example: + errors: [ ] + description: | + Successfully uploaded or validated the component rules file. The returned response object will not contain any errors. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "422": + $ref: '#/components/responses/422-rules' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossier-status-definitions: + get: + summary: Returns the list of all existing dossier status definitions + tags: + - 1. Dossier Templates + description: | + Retrieves a collection of dossier status definitions associated with a specific dossier template. Each dossier + status definition includes details such as the status name, description, and other relevant metadata. This endpoint + is useful for clients needing to display or set the status of a dossier associated with a specific dossier template. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + content: + '*/*': + schema: + $ref: '#/components/schemas/DossierStatusDefinitionList' + description: | + Successfully returned the dossier status definitions for the specified dossier template. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossier-attribute-definitions: + get: + summary: Returns the list of all existing dossier attribute definitions + tags: + - 1. Dossier Templates + description: | + Retrieves a collection of dossier attribute definitions associated with a specific dossier template. Each dossier + attribute definition includes details such as attribute type, name, and other relevant metadata. This endpoint + is useful for clients needing to understand what attributes are expected or allowed for dossiers associated with + a specific dossier template. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + content: + '*/*': + schema: + $ref: '#/components/schemas/DossierAttributeDefinitionList' + description: | + Successfully returned the dossier attribute definitions for the specified dossier template. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/file-attribute-definitions: + get: + summary: Returns the list of all existing file attribute definitions + tags: + - 1. Dossier Templates + description: | + Retrieves a collection of file attribute definitions associated with a specific dossier template. Each file + attribute definition includes details such as attribute type, name, and other relevant metadata. This endpoint + is useful for clients needing to understand what attributes are expected or allowed for files associated with + a specific dossier template. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + content: + '*/*': + schema: + $ref: '#/components/schemas/FileAttributeDefinitionList' + description: | + Successfully returned the file attribute definitions for the specified dossier template. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/component-mappings: + get: + operationId: listAllMappings + tags: + - 1. Dossier Templates + summary: Returns a list of all existing component mappings in a dossier template + description: | + Use this endpoint to retrieve a summary of all component mappings associated with a specific DossierTemplate. + + The summary consists of the stored metadata of a component mapping file. + This endpoint is useful for clients to understand the available mappings under a particular DossierTemplate. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMappingSummary' + description: | + Successfully returned the component mapping summary for the specified dossier template. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + post: + operationId: uploadMapping + summary: Upload a new component mapping to a DossierTemplate. + description: | + Use this endpoint to upload a new component mapping to a specific DossierTemplate. + + #### File Requirements + - **Format:** The file must be in CSV (comma-separated values) format. + - **Header Row:** The first row should contain the column labels. + - **Data Consistency:** All rows must have the same number of columns to ensure rectangular data structure. + + #### Sorting and Performance + - **Sorting:** Rows are automatically sorted by the values in each column, from left to right, to enhance lookup speed. + - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance. + + #### Customization Options + Users can specify the delimiter and encoding used in the CSV file. + + #### Usage + The component mapping file can be utilized in component rules to relate components to existing master data. + + #### Example + + | search_value | mapped_value | + |--------------|--------------| + | Alice | Manager | + | Bob | Developer | + | Charlie | Analyst | + + + tags: + - 1. Dossier Templates + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/mappingName' + - $ref: '#/components/parameters/encoding' + - $ref: '#/components/parameters/delimiter' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMapping' + description: | + Component mapping uploaded successfully and returned the component mapping metadata. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + + /api/dossier-templates/{dossierTemplateId}/component-mappings/{comonentMappingId}: + get: + operationId: downloadMappingFile + tags: + - 1. Dossier Templates + summary: Download a specific component mapping file of a specific dossier template. + description: | + Use this endpoint to download a specific component mapping file of a designated DossierTemplate. + + - The file retains its original name and encoding as when it was uploaded. + - The sorting of the file may have changed to enable faster lookups. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + responses: + "200": + headers: + Content-Disposition: + schema: + type: string + example: attachment; filename*=mapping.csv + content: + text/plain: + schema: + type: string + description: | + Successfully downloaded the requested component mapping. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + put: + operationId: updateMapping + summary: Update an existing component mapping of a DossierTemplate. + description: | + Use this endpoint to update an existing component mapping of a specific dossier template. + + #### File Requirements + - **Format:** The file must be in CSV (comma-separated values) format. + - **Header Row:** The first row should contain the column labels. + - **Data Consistency:** All rows must have the same number of columns to ensure rectangular data structure. + + #### Sorting and Performance + - **Sorting:** Rows are automatically sorted by the values in each column, from left to right, to enhance lookup speed. + - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance. + + #### Customization Options + Users can specify the delimiter and encoding used in the CSV file. + + #### Usage + The component mapping file can be utilized in component rules to relate components to existing master data. + tags: + - 1. Dossier Templates + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + - $ref: '#/components/parameters/encoding' + - $ref: '#/components/parameters/delimiter' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ComponentMapping' + description: | + Component mapping uploaded successfully and returned the component mapping metadata. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + delete: + operationId: deleteMappingFile + tags: + - 1. Dossier Templates + summary: Delete a specific component mapping file of a specific dossier template. + description: | + Utilize this endpoint to delete a specific component mapping file of a designated dossier template. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/componentMappingId' + responses: + "204": + description: | + Successfully deleted the requested component mapping. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-mapping' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers: + get: + operationId: getDossiers + tags: + - 2. Dossiers + summary: Retrieve a list of dossiers based on a specific dossier template. + description: | + By default, this endpoint provides a list of all *active* dossiers that are based on a specific dossier template. + + It returns an object containing an array of dossier objects under the key `dossiers`. Each dossier object contains + an identifier, a name, a description, an owner, and other fields with further details about the dossier. + + Use this endpoint to fetch the required dossiers before performing actions on specific ones. Use the query parameters + to modify the response. E.g., set the `includeArchivedDossiers` parameter to `true` so that the response also contains + *archived* dossiers. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/includeActiveDossiers' + - $ref: '#/components/parameters/includeArchivedDossiers' + - $ref: '#/components/parameters/includeSoftDeletedDossiers' + responses: + "200": + content: + '*/*': + schema: + $ref: '#/components/schemas/DossierList' + description: | + Successfully retrieved the dossiers that are based on the requested dossier template. + + The response could be an empty + list if there are no *active* dossiers based on it. If query parameters have been set, an empty list will be returned if + no dossier matches the respective combination. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + post: + operationId: createOrUpdateDossier + tags: + - 2. Dossiers + summary: Creates or updates a dossier for a specific dossier template. + description: | + This endpoint is meant to create a new dossier or to update an existing one. + + To create a new dossier, do not set the identifier of the dossier object provided in the request body. + + To update an existing dossier, the set identifier needs to be valid. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DossierRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/Dossier' + description: | + Successfully saved the dossier. + "201": + content: + application/json: + schema: + $ref: '#/components/schemas/Dossier' + description: | + successfully created the dossier. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "409": + $ref: '#/components/responses/409-dossier-conflict' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}: + delete: + operationId: deleteDossier + tags: + - 2. Dossiers + summary: Deletes a specific dossier by its identifier. + description: | + Removes a dossier either in a recoverable manner (soft-delete) or permanently. By default, a soft-deleted dossier + remains recoverable for a retention period determined by application settings. The default retention period is + 96 hours (4 days) but may vary based on configuration. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/permanentlyDeleteDossier' + responses: + "204": + description: | + Dossier deletion successful. Note that the absence of the dossier with the given `dossierId` is confirmed, + regardless of whether it was pre-existing, already removed, or never present. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + get: + operationId: getDossier + tags: + - 2. Dossiers + summary: Retrieve a specific dossier by its identifier. + description: | + Utilize this endpoint to retrieve comprehensive details about a designated dossier. The response encompasses various + attributes such as the dossier's unique ID, name, description, associated owner, among other pertinent details. + + If the dossier has been soft-deleted but not yet permanently deleted, the `includeSoftDeleted` parameter can be utilized + to get it. Without this, a soft-deleted dossier would return a `404 Dossier not found` error. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/includeSoftDeletedDossier' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/Dossier' + description: | + Successfully retrieved the requested dossier. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/attributes: + post: + operationId: setDossierAttributes + tags: + - 2. Dossiers + summary: Update or set attributes for a specific dossier. + description: | + This endpoint facilitates the updating or setting of specific dossier attributes for a given dossier. + Ensure you provide the necessary dossier attributes within the request body. + + Use this route to maintain or enhance dossier metadata and properties. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DossierAttributes' + required: true + responses: + "204": + description: | + Dossier attributes successfully updated. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/create-download: + post: + operationId: prepareDossierDownload + tags: + - 2. Dossiers + summary: Initiate the creation of a download package for all files of a dossier. + description: | + To download files and reports, a download package needs to be prepared. This endpoint + facilitates to define the content of the download package and to start the creation of it for all + files of the dossier. The response of this endpoint contains an identifier for the download that + is needed to query the status until the download package is prepared, and finally to actually + download the file once it is available. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadStatus' + description: | + Successfully started the creation of the download package for all files of the requested dossier. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files: + get: + operationId: getDossierStatus + tags: + - 3. Files + summary: Retrieves the status of the files of a specific dossier. + description: | + This endpoint provides status information for each file within a designated dossier. + + The file status encompasses attributes such as identifier, file name, number of pages, file size, + processing and workflow status, among other pertinent details. By default, only active files + are fetched. However, various parameters can be combined to adjust the scope of the response. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/includeActiveFiles' + - $ref: '#/components/parameters/includeArchivedFiles' + - $ref: '#/components/parameters/includeSoftDeletedFiles' + responses: + "200": + content: + '*/*': + schema: + $ref: '#/components/schemas/FileStatusList' + description: | + Successfully retrieved file status information. + + The response could be an empty list if the dossier contains no *active* files. If query parameters + have been set, an empty list will be returned if no file matches the respective combination. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + post: + operationId: upload + tags: + - 3. Files + summary: Upload a file and associate it with a specific dossier. + description: | + This endpoint facilitates the upload of files to a specific dossier. + + Upon successful upload, the system returns the unique identifier of the uploaded file. In cases where the dossier already + contains a file with the same name, the new file will replace the existing one. + + Supported file formats include PDF documents, Microsoft Office documents (if supported by your license), CSV files, and + ZIP archives. + + Notably, Microsoft office files (file extensions `docx`, `xlsx`, or `pptx`) are automatically converted into PDF format + upon upload. Be aware that due to potential font discrepancies, the resulting PDF may slightly differ in appearance from + the original. Nevertheless, our system ensures maximum fidelity to the original content. + + CSV files offer support for bulk importing of file attributes. If the dossier is set up to map specific CSV fields to + file attributes and the uploaded CSV aligns with this mapping, the system will process the CSV accordingly. Mismatched + CSVs will be disregarded. For successfully processed CSV files, the response contains information about the file attributes + and the annotated files. + + ZIP archives offer support to upload multiple files at once. In case of an upload of a ZIP archive, the system returns a + list of unique identifiers of the contained files. The archive can even contain a mix of files like some PDF files and + a CSV file that will annotate some file attributes. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/keepManualChanges' + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadRequest' + responses: + "201": + content: + application/json: + schema: + $ref: '#/components/schemas/FileUploadResult' + description: | + Successfully uploaded the file. Returns an object with the unique identifier of the uploaded file, a list of unique + identifiers in case of an uploaded ZIP archive, or some annotation information in case of a successfully processing + of an uploaded CSV file. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/{fileId}: + delete: + operationId: deleteFile + tags: + - 3. Files + summary: Deletes a specific file associated with a dossier and its template. + description: | + This endpoint facilitates the removal of a file linked to a specific dossier under a dossier template. By default, the + file undergoes a soft-delete, meaning it can be restored within the retention period determined by application settings. + The default retention period is 96 hours (4 days) but may vary based on configuration. Use the `deletePermanently` query + parameter to opt for a permanent deletion, preventing any future restoration. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/fileId' + - $ref: '#/components/parameters/permanentlyDeleteFile' + responses: + "204": + description: | + File deletion successful. This confirms the absence of the specified file, irrespective of its previous existence. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + get: + operationId: getFileStatus + tags: + - 3. Files + summary: Retrieves the status of a particular file. + description: | + This endpoint is designed to fetch the status of a specific file associated to a dossier based on a dossier template. If + the file has been soft-deleted but not yet permanently deleted, the `includeSoftDeleted` parameter can be utilized to get + the status of such files. Without this, a soft-deleted file would return a `404 File not found` error. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/fileId' + - $ref: '#/components/parameters/includeSoftDeletedFile' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/FileStatus' + description: | + Successfully retrieved the status of the requested file. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/{fileId}/create-download: + post: + operationId: prepareFileDownload + tags: + - 3. Files + summary: Initiate the creation of a download package for a single file of a dossier. + description: | + To download files and reports, a download package needs to be prepared. This endpoint + facilitates to define the content of the download package and to start the creation of it for the + file. The response of this endpoint contains an identifier for the download that is needed to + query the status until the download package is prepared for download, and finally to actually + download the file once it is available. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/fileId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadStatus' + description: | + Successfully started the creation of the download package for the requested file. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/bulk/delete: + post: + operationId: deleteFiles + tags: + - 3. Files + summary: Bulk delete specific files within a dossier. (DELETE with body payload) + description: | + This endpoint allows for the bulk deletion of specified files associated with a certain dossier. + + It provides the flexibility to perform either a soft-delete (by default) or a permanent deletion. A soft-deleted + file remains restorable within the retention period set by the application (96h by default). In contrast, permanently + deleted files are irrevocably removed and cannot be restored. + + Use this route to manage large file deletions efficiently. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/permanentlyDeleteFiles' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/FileDeleteRequest' + required: true + responses: + "204": + description: | + Bulk file deletion successful. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/bulk/create-download: + post: + operationId: prepareBulkDownload + tags: + - 3. Files + summary: Initiate the creation of a download package for specific files within a dossier. + description: | + To download files and reports, a download package needs to be prepared. This endpoint + facilitates to define the content of the download package and to start the creation of it for + multiple files of a dossier. The response of this endpoint contains an identifier for the download + that is needed to query the status until the download package is prepared for download, and finally + to actually download the file once it is available. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BulkDownloadRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadStatus' + description: | + Successfully started the creation of the download package for the requested files. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/{fileId}/attributes: + post: + operationId: setFileAttributes + tags: + - 3. Files + summary: Update or set attributes for a specific file. + description: | + This endpoint facilitates the updating or setting of specific file attributes for a given file within a dossier. + Ensure you provide the necessary file attributes within the request body. + + Use this route to maintain or enhance file metadata and properties. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/fileId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/FileAttributes' + required: true + responses: + "204": + description: | + File attributes successfully updated. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/{fileId}/components: + get: + operationId: getComponents + tags: + - 4. Components + summary: Fetch the components associated with a specific file. + description: | + This endpoint retrieves the derived and extracted components from a given file within a dossier. Components represent + various aspects, metadata or content of the file. Entity and component rules define these components based on the file's + content. They can give a *structured view* on a document's text. + + To include detailed component information, set the `includeDetails` query parameter to `true`. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/fileId' + - $ref: '#/components/parameters/includeComponentDetails' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/FileComponents' + application/xml: + schema: + $ref: '#/components/schemas/FileComponents' + description: | + Successfully retrieved file components. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/bulk/get-components: + get: + operationId: getComponentsOfDossier + tags: + - 4. Components + summary: Returns the RSS response for all files of a dossier + description: | + This endpoint fetches components for all files associated with a specified dossier. Like individual file components, + these represent various aspects, metadata or content of the files. Entity and component rules define these components based on the file's + content. They can give a *structured view* on a document's text. + + To include detailed component information, set the `includeDetails` query parameter to `true`. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/fileId' + - $ref: '#/components/parameters/includeComponentDetails' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/FileComponentsList' + application/xml: + schema: + $ref: '#/components/schemas/FileComponentsList' + description: | + Successfully fetched components for all files in the dossier. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/download: + get: + operationId: getDownloadStatusList + tags: + - 5. Downloads + summary: Get the list of downloads for the current user + description: | + This endpoint facilitates to retrieve the list of downloads for the current user. The response contains + status objects that represent each download and provide information on whether the download is still + in preparation, is already available or whether an error occurred when the download package was created. + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadStatusList' + description: List of all downloads of the current users successfully retrieved. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/downloads/{downloadId}: + get: + operationId: getDownloadStatus + tags: + - 5. Downloads + summary: Get the status for a specific download of the current user + description: | + This endpoint facilitates to retrieve the status for a specific download. In addition to other + information, the status indicates whether the download is still being prepared, is already + available, or whether an error occurred when the download package was created. + parameters: + - $ref: '#/components/parameters/downloadId' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadStatus' + description: Status of the download successfully retrieved. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-download' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + delete: + operationId: deleteDownload + tags: + - 5. Downloads + summary: Deletes a specific download. + description: | + This endpoint facilitates the removal of a download. + parameters: + - $ref: '#/components/parameters/downloadId' + responses: + "204": + description: | + Download deletion successful. This confirms the absence of the specified download, irrespective of its previous existence. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-download' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/downloads/{downloadId}/download: + get: + operationId: download + tags: + - 5. Downloads + - persistence-service + summary: Download the download package. + description: | + This endpoint facilitates to actually download the created download package. The request will + only be successful if the status of the download is `READY`. + parameters: + - $ref: '#/components/parameters/downloadId' + responses: + "200": + headers: + Content-Disposition: + schema: + type: string + example: attachment; filename*=utf-8''example.zip + content: + application/octet-stream: + schema: + type: string + format: binary + description: Successfully downloaded the requested file. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-download' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/users: + get: + operationId: getUsers + tags: + - 6. Users + - tenant-user-management + summary: Get a list of users + description: | + This endpoint facilitates to retrieve a list of known users. + + With the `username` parameter you can filter for a specific user name. If the parameter is + used, the returned list either contains a single matching entry or is empty. + parameters: + - $ref: '#/components/parameters/username' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/UserList' + description: List of users successfully retrieved. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/users/{userId}: + get: + operationId: getUserById + tags: + - 6. Users + summary: Retrieve a specific user by its identifier. + description: | + This endpoint facilitates to retrieve a specific user. + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/User' + description: User successfully retrieved. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-user' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/license/active/usage: + post: + operationId: getLicenseReport + tags: + - 7. License + summary: Generate and retrieve a license usage report. + description: | + This endpoint enables users to create and fetch a report detailing the active usage of licenses. The report contains + valuable insights and metrics that can aid in monitoring and managing license consumption. Ensure your request body + contains the necessary parameters for generating this report. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/LicenseReportRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/LicenseReport' + description: | + License report successfully generated and retrieved. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' +components: + headers: + Authorization: + description: JWT token for authorization. Format should be `Bearer {JWT_TOKEN}`. + required: true + schema: + type: string + example: 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI...' + responses: + "400": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + The request was malformed or invalid. Double-check the request structure or parameters. + "401": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Unauthorized access. You must provide valid authentication credentials to access this + resource. This error can happen if the access token was revoked or has expired. + "403": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Forbidden. Your credentials are valid, but you do not have permission to access this resource. + "404-dossier-template": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Dossier template not found. This happens if the requested dossier template does not exist. + "404-mapping": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Dossier template or component mapping not found. This happens if the requested dossier template or component mapping does not exist. + "404-dossier": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Dossier not found. This happens if the dossier template or the dossier is deleted or did never exist. + + Some endpoints support a `includeSoftDeleted` parameter: If this is set to `true`, this response is returned + for a previously existing dossier only if it is actually deleted permanently. + "404-file": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + File not found. This indicates that the dossier, dossier template, or file is deleted or did never exist. + + Some endpoints support a `includeSoftDeleted` parameter: If this is set to `true`, this response is returned + only if the file is deleted permanently. + "404-download": + content: + '*/*': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Download not found. This happens if the requested download does not exist for the current user. + "404-user": + content: + '*/*': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + User not found. This happens if the requested user does not exist. + "409-dossier-conflict": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Name conflict: The provided name is already in use by another dossier. It needs to be unique in the scope of your workspace. + 422-rules: + content: + 'application/json': + schema: + $ref: '#/components/schemas/RuleValidation' + description: | + Invalid rules file: There were validation errors, the rules file is unprocessable. + "429": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Too many requests have been made in a given time frame. Rate limiting is in place to prevent abuse. + "500": + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Internal Server Error. An unexpected error occurred on the server side. Please try again later or contact support. + parameters: + dossierTemplateId: + name: dossierTemplateId + in: path + required: true + schema: + type: string + style: simple + explode: false + description: The identifier of a dossier template + componentMappingId: + name: componentMappingId + in: path + required: true + schema: + type: string + style: simple + explode: false + description: The identifier of a component mapping + dryRun: + name: dryRun + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A toggle to activate the dry-run mode: If set to `false` (default), the request will update the system. + If set to `true`, the request will just be evaluated without actual changes in the system. + encoding: + name: encoding + required: false + in: query + schema: + type: string + enum: + - UTF-8 + - UTF-16 + - UTF_16BE + - UTF_16LE + - ISO-8859-1 + - US-ASCII + example: UTF-8 + default: UTF-8 + description: "An identifier for the used encoding of the file. Only java's standard charsets are supported." + delimiter: + name: delimiter + required: false + in: query + schema: + type: string + minLength: 1 + maxLength: 1 + example: ',' + default: ',' + description: "The delimiter used as a separator in a csv file." + mappingName: + name: name + required: false + in: query + schema: + type: string + example: "MasterDataMapping" + description: "The name with which the mapping should be associated with. If none is provided, the file name will be used." + dossierId: + name: dossierId + in: path + required: true + schema: + type: string + style: simple + explode: false + description: The identifier of a dossier + fileId: + name: fileId + in: path + required: true + schema: + type: string + style: simple + explode: false + description: The identifier of a file + includeActiveDossiers: + name: includeActive + in: query + required: false + schema: + default: true + type: boolean + style: form + explode: true + description: | + A toggle to include or exclude active dossiers: If `true` (default), dossiers + that are neither archived nor soft-deleted are included. If `false`, they are + ignored. + includeActiveFiles: + name: includeActive + in: query + required: false + schema: + default: true + type: boolean + style: form + explode: true + description: | + A toggle to include or exclude active files: If set to `true` (default), files + that neither belong to an archived dossier nor have been soft-deleted are + included. If set to `false`, they are ignored. + includeArchivedDossiers: + name: includeArchived + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A toggle to include or exclude archived dossiers: If set to `true`, archived + dossiers are included. If set to `false` (default), they are ignored. + includeArchivedFiles: + name: includeArchived + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A toggle to include or exclude archived files (i.e., files of archived dossiers): + If set to `true`, files of archived dossiers are included. If set to `false` + (default), they are ignored. + includeSoftDeletedDossiers: + name: includeSoftDeleted + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A toggle to include soft-deleted dossiers: + - `true`: The response will include soft-deleted dossiers. + - `false` (default): Soft-deleted dossiers are ignored and will not be part of the response. + + Soft-deleted means that the dossier has not been deleted permanently. + includeSoftDeletedDossier: + name: includeSoftDeleted + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A Toggle to return a soft-deleted dossier: + - `true`: The system returns the dossier regardless whether it has been soft-deleted or not. + - `false` (default): The system returns only non-deleted dossiers. It would return a `404 Dossier not found` error + if the requested dossier has been soft-deleted. + + Soft-deleted means that the dossier has not been deleted permanently. + includeSoftDeletedFiles: + name: includeSoftDeleted + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A toggle to include soft-deleted files: + - `true`: The response will include soft-deleted files. + - `false` (default): Soft-deleted files are ignored and will not be part of the response. + + Soft-deleted means that the file or dossier of the file has not been deleted permanently. + includeSoftDeletedFile: + name: includeSoftDeleted + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A Toggle to return the status of soft-deleted files: + - `true`: The system returns the status of the file regardless whether it has been soft-deleted or not. + - `false` (default): The system returns only non-deleted files. It would return a `404 File not found` error + if the requested file has been soft-deleted. + + Soft-deleted means that the file or dossier of the file has not been deleted permanently. + permanentlyDeleteDossier: + name: deletePermanently + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A toggle to determine the deletion mode for a dossier: + - `true`: The dossier will be permanently deleted and can't be restored. + - `false` (default): Soft-delete, allowing restoration within the application-configured retention period. + + Using this parameter, you can also permanently delete a previously soft-deleted dossier during its retention period. + + Note: Deleting a dossier also deletes all associated files. + permanentlyDeleteFile: + name: deletePermanently + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + Determines the deletion mode for a file. + - `true`: The file will be permanently deleted and can't be restored. + - `false` (default): Soft-delete, allowing restoration within the retention period, provided its parent dossier + hasn't been deleted. If the parent dossier has been deleted meanwhile, it needs to be restored first. + + Using this parameter, you can also permanently delete a previously soft-deleted file during its retention period. + permanentlyDeleteFiles: + name: deletePermanently + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + Determines the deletion mode for a list of files: + - `true`: The files will be permanently deleted and can't be restored. + - `false` (default): Soft-delete, allowing restoration within the retention period, provided the files' parent dossier + hasn't been deleted. If the parent dossier has been deleted meanwhile, it needs to be restored first. + + Using this parameter, you can also permanently delete a previously soft-deleted files during their retention period. + keepManualChanges: + name: keepManualChanges + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A toggle to keep manual changes, i.e., to preserve manually added annotations or manipulations on existing ones when + overwriting a file: + - `true`: The system keeps the manual changes on re-uploading (overwriting) the file. + - `false` (default): Manual changes get discarded on re-uploading the file. + includeComponentDetails: + name: includeDetails + in: query + required: false + schema: + default: false + type: boolean + style: form + explode: true + description: | + A toggle to decide whether to include detailed component information in the response: + - `true`: The component object's field `componentDetails` stores detailed information about the *source* of its + respective value(s). + - `false` (default): The component object does not contain a field `componentDetails`. + downloadId: + name: downloadId + in: path + required: true + schema: + type: string + style: simple + explode: false + description: The identifier for the file to download. + username: + name: username + in: query + required: false + schema: + type: string + style: form + explode: true + description: | + If the `username` parameter is set, the user list is filtered for that specific user name. This means the list + either has one matching entry or is empty. + schemas: + EntityReference: + type: object + description: | + Represents a reference to an entity object discovered in a document. Together with the unique identifier + of the entity, the reference contains some basic details of the entity. + properties: + id: + type: string + description: | + The unique identifier for the entity. + + Note: In general, it is a valid UUID. Only if an entity spans over multiple pages, it is split on page breaks. Each + part becomes an own entity object and the first one keeps the UUID while the following get an additional suffix. + + Example: Even is `bcd22239-c3df-442f-a5a1-1664cba94dc6_2` is not a valid UUID it is still a valid identifier for an + entity object. + entityRuleId: + type: string + description: An identifier that represents a specific rule associated with the entity. + type: + type: string + description: The name of the entity. + page: + type: integer + format: int32 + description: The page number in the document where the entity is located. + example: + id: bcd22239-cedf-442f-a5a1-1664cba94dc6 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + Component: + type: object + description: Represents a component with its associated values. + properties: + name: + type: string + description: The name of the component. + componentValues: + type: array + description: A list of value objects associated with the component. + items: + $ref: '#/components/schemas/ComponentValue' + example: + name: the component name + componentValues: + - value: my overwritten component value + originalValue: the original value + componentRuleId: COM.0.0 + valueDescription: My value description + entityReferences: + - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + - id: bcd22239-c3df-442f-a5a1-1664cba94dc6_2 + entityRuleId: ABC.0.0 + type: entity_type + page: 124 + - id: b748b89a-5679-4254-9286-1dd652d9970b + entityRuleId: DEF.13.37 + type: another_entity_type + page: 456 + - value: yet another component value + originalValue: yet another component value + componentRuleId: COM.0.1 + valueDescription: Another value description + entityReferences: + - id: 70496456-a016-4679-81b1-6c8856dded6e + entityRuleId: XYZ.0.0 + type: yet_another_entity_type + page: 123 + ComponentValue: + type: object + description: Describes a value associated with a component, including its origin and related entities found in the document. + properties: + value: + type: string + description: The current value of the component. + originalValue: + type: string + description: The original value before any modifications. + componentRuleId: + type: string + description: Identifier for the rule associated with this component value. + valueDescription: + type: string + description: A brief description of the value. + entityReferences: + items: + $ref: '#/components/schemas/EntityReference' + type: array + example: + value: my overwritten component value + originalValue: the original value + componentRuleId: COM.0.0 + valueDescription: My value description + entityReferences: + - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + - id: b748b89a-5679-4254-9286-1dd652d9970b + entityRuleId: DEF.13.37 + type: another_entity_type + page: 456 + FileComponents: + type: object + description: Represents file details along with its associated components and values. + properties: + dossierTemplateId: + format: uuid + type: string + description: Identifier for the template associated with the dossier. + dossierId: + format: uuid + type: string + description: Identifier for the dossier. + fileId: + type: string + description: Identifier for the file. + filename: + type: string + description: Name of the file. + components: + type: object + description: A map of component names to their list of values. + additionalProperties: + items: + type: string + type: array + componentDetails: + type: object + description: A map of component names to their detailed representations. + additionalProperties: + $ref: '#/components/schemas/Component' + example: + dossierTemplateId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 + dossierId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 + fileId: 1fdbd888b39059c8cf171df26f62f8a5 + filename: MyFile.pdf + components: + "my component": + - my overwritten component value + - yet another component value + "yet another component": + - only one value + componentDetails: + - name: my component + componentValues: + - value: my overwritten component value + originalValue: the original value + componentRuleId: COM.0.0 + valueDescription: My value description + entityReferences: + - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + - id: b748b89a-5679-4254-9286-1dd652d9970b + entityRuleId: DEF.13.37 + type: another_entity_type + page: 456 + - value: yet another component value + originalValue: yet another component value + componentRuleId: COM.0.1 + valueDescription: Another value description + entityReferences: + - id: 70496456-a016-4679-81b1-6c8856dded6e + entityRuleId: XYZ.0.0 + type: yet_another_entity_type + page: 123 + - name: yet another component + componentValues: + - value: only one value + originalValue: only one value + componentRuleId: COM.0.0 + valueDescription: My value description + entityReferences: + - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + - id: b748b89a-5679-4254-9286-1dd652d9970b + entityRuleId: DEF.13.37 + type: another_entity_type + page: 456 + FileComponentsList: + type: object + description: A list of files and their associated components. + properties: + files: + type: array + description: List of files with their component details. + items: + $ref: '#/components/schemas/FileComponents' + example: + files: + - dossierTemplateId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 + dossierId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 + fileId: 1fdbd888b39059c8cf171df26f62f8a5 + filename: MyFile.pdf + components: + "my component": + - my overwritten component value + - yet another component value + "yet another component": + - only one value + componentDetails: + - name: my component + componentValues: + - value: my overwritten component value + originalValue: the original value + componentRuleId: COM.0.0 + valueDescription: My value description + entityReferences: + - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + - id: b748b89a-5679-4254-9286-1dd652d9970b + entityRuleId: DEF.13.37 + type: another_entity_type + page: 456 + - value: yet another component value + originalValue: yet another component value + componentRuleId: COM.0.1 + valueDescription: Another value description + entityReferences: + - id: 70496456-a016-4679-81b1-6c8856dded6e + entityRuleId: XYZ.0.0 + type: yet_another_entity_type + page: 123 + - name: yet another component + componentValues: + - value: only one value + originalValue: only one value + componentRuleId: COM.0.0 + valueDescription: My value description + entityReferences: + - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + - id: b748b89a-5679-4254-9286-1dd652d9970b + entityRuleId: DEF.13.37 + type: another_entity_type + page: 456 + - dossierTemplateId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 + dossierId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 + fileId: 4d2334def5fced0003888e47cbc270f7 + filename: Copy of MyFile.pdf + components: + "my component": + - my overwritten component value + - yet another component value + "yet another component": + - only one value + componentDetails: + - name: my component + componentValues: + - value: my overwritten component value + originalValue: the original value + componentRuleId: COM.0.0 + valueDescription: My value description + entityReferences: + - id: dadb54d4-587b-4e69-8c07-be9446a33537 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + - id: 30339f5a-4c19-447f-b0a0-0f14c094037e + entityRuleId: DEF.13.37 + type: another_entity_type + page: 456 + - value: yet another component value + originalValue: yet another component value + componentRuleId: COM.0.1 + valueDescription: Another value description + entityReferences: + - id: 086cd500-97da-44f8-8628-3324d11b4e8d + entityRuleId: XYZ.0.0 + type: yet_another_entity_type + page: 123 + - name: yet another component + componentValues: + - value: only one value + originalValue: only one value + componentRuleId: COM.0.0 + valueDescription: My value description + entityReferences: + - id: dadb54d4-587b-4e69-8c07-be9446a33537 + entityRuleId: ABC.0.0 + type: entity_type + page: 123 + - id: 30339f5a-4c19-447f-b0a0-0f14c094037e + entityRuleId: DEF.13.37 + type: another_entity_type + page: 456 + DossierStatusDefinition: + type: object + description: | + The `DossierStatusDefinition` object contains the relevant information to define a dossier status. The dossier status + is used to assign a custom status to a dossier. + properties: + id: + type: string + format: uuid + description: | + A unique identifier for the dossier status definition. This ID is automatically generated by + the system upon creation and is used for referencing the dossier status definition in API calls. + example: bcd22239-cedf-442f-a5a1-1664cba94dc6 + name: + type: string + description: | + User-defined name of the dossier status definition, capturing its essence. The name needs to be unique + for the dossier template. + example: "Done" + description: + type: string + description: | + A text that can be added to provide further details about the status. E.g., what it is intended for or + the circumstances under which it can be used. + example: "Dossiers with this status should only contain approved files and indicate that the users have completed the component extractions." + rank: + format: int32 + type: integer + description: | + A number that allows to define a custom display order. + default: "" + example: 1 + color: + type: string + description: | + A hexadecimal color code that can be set to assign a color to a + the `PREVIEW` file. + + - Yellow is `#ffda05` + - Green is `#5eb160` + example: "#5eb160" + required: + - name + - rank + - color + DossierAttributeDefinition: + type: object + description: | + The `DossierAttributeDefinition` object contains the relevant information to define a dossier attribute. Dossier attributes + are used to manage additional meta-data of dossiers. + properties: + id: + type: string + format: uuid + description: | + A unique identifier for the dossier attribute definition. This ID is automatically generated by + the system upon creation and is used for referencing the dossier attribute definition in API calls. + name: + type: string + description: | + User-defined name of the dossier attribute definition, capturing its essence. The name needs to be unique + for the dossier template. + type: + type: string + enum: + - TEXT + - NUMBER + - DATE + description: | + Determines the type of the dossier attribute's value. Please note that currently the system + does not validate the values against this definition. This is just a hint for a user interface + that needs to handle invalid entries. Possible values for the type: + - `TEXT`: The value is just a string, i.e., any sequence of characters. + - `NUMBER`: The value is a string expressing a number, with or without decimals. + - `DATE`: The value is a string expressing a date information. + reportingPlaceholder: + type: string + description: | + The name of the placeholder of the dossier attribute that can be used in report templates. The + placeholder follows a specific format convention: + `{{dossier.attribute.}}` while the name is transformed into 'PascalCase' and does not contain + whitespaces. The placeholder is unique in a dossier template. + required: + - name + - type + example: + id: "123e4567-e89b-12d3-a456-426614174000" + name: "Document Summary" + type: "TEXT" + reportingPlaceholder: "{{dossier.attribute.DocumentSummary}}" + FileAttributeDefinition: + type: object + description: | + The `FileAttributeDefinition` object contains the relevant information to define a file attribute. File attributes + are used to manage additional meta-data of files. + properties: + id: + type: string + format: uuid + description: | + A unique identifier for the file attribute definition. This ID is automatically generated by + the system upon creation and is used for referencing the file attribute definition in API calls. + name: + type: string + description: | + User-defined name of the file attribute definition, capturing its essence. The name needs to be unique + for the dossier template. + type: + type: string + enum: + - TEXT + - NUMBER + - DATE + description: | + Determines the type of the file attribute's value. Please note that currently the system + does not validate the values against this definition. This is just a hint for a user interface + that needs to handle invalid entries. Possible values for the type: + - `TEXT`: The value is just a string, i.e., any sequence of characters. + - `NUMBER`: The value is a string expressing a number, with or without decimals. + - `DATE`: The value is a string expressing a date information. + mappedCsvColumnHeader: + type: string + description: | + The name of a CSV column. When importing a CSV file with additional file information the system gets the value from the respective CSV column. + reportingPlaceholder: + type: string + description: | + The name of the placeholder of the file attribute that can be used in report templates. The placeholder follows a specific format convention: + `{{file.attribute.}}` while the name is transformed into 'PascalCase' and does not contain whitespaces. The placeholder is unique in a dossier template. + displaySettings: + $ref: '#/components/schemas/FileAttributeDisplaySettings' + required: + - name + - type + example: + id: "123e4567-e89b-12d3-a456-426614174000" + name: "Document Type" + type: "TEXT" + mappedCsvColumnHeader: "DocumentCategory" + reportingPlaceholder: "{{file.attribute.DocumentType}}" + displaySettings: + primaryAttribute: true + editable: true + filterable: true + displayedInFileList: false + FileAttributeDisplaySettings: + type: object + description: | + Display setting for the DocuMine user interface. These settings control how the UI handles and presents the file attributes. + properties: + primaryAttribute: + type: boolean + description: | + If `true`, the DocuMine user interfaces show the value of the file attribute in the file list below the file name. + editable: + type: boolean + description: | + If `true`, the DocuMine user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible. + filterable: + type: boolean + description: | + If `true`, the DocuMine user interfaces add filter options to the file list. + displayedInFileList: + type: boolean + description: | + if `true`, the DocuMine user interfaces show the values in the file list. + required: + - primaryAttribute + - editable + - filterable + - displayedInFileList + example: + primaryAttribute: false + editable: true + filterable: true + displayedInFileList: false + DossierStatusDefinitionList: + type: object + description: A list of dossier status definitions. + properties: + dossierStatusDefinitions: + items: + $ref: '#/components/schemas/DossierStatusDefinition' + type: array + example: + dossierStatusDefinitions: + - id: "123e7567-e89b-12d3-a456-426614174000" + name: "In Progress" + description: "Dossiers with this status are currently being processed by the users." + rank: 0 + color: "#ffda05" + - id: "23e45378-e90b-12d3-a456-765114174321" + name: "Done" + description: "Dossiers with this status should only contain approved files and indicate that the users have completed the component extraction." + rank: 1 + color: "#5eb160" + DossierAttributeDefinitionList: + type: object + description: A list of dossier attribute definitions. + properties: + dossierAttributeDefinitions: + items: + $ref: '#/components/schemas/DossierAttributeDefinition' + type: array + example: + dossierAttributeDefinitions: + - id: "123e4567-e89b-12d3-a456-426614174000" + name: "Dossier Summary" + type: "TEXT" + reportingPlaceholder: "{{dossier.attribute.DossierSummary}}" + - id: "23e45678-e90b-12d3-a456-765114174321" + name: "Comment" + type: "TEXT" + reportingPlaceholder: "{{dossier.attribute.Comment}}" + FileAttributeDefinitionList: + type: object + description: A list of file attribute definitions. + properties: + csvImportSettings: + $ref: '#/components/schemas/CsvImportSettings' + fileAttributeDefinitions: + items: + $ref: '#/components/schemas/FileAttributeDefinition' + type: array + example: + csvImportSettings: + csvMappingActive: true + filenameMappingCsvColumnHeader: "Filename" + delimiter: "," + encoding: "UTF-8" + fileAttributeDefinitions: + - id: "123e4567-e89b-12d3-a456-426614174000" + name: "Document Type" + type: "TEXT" + mappedCsvColumnHeader: "DocumentCategory" + reportingPlaceholder: "{{file.attribute.DocumentType}}" + displaySettings: + primaryAttribute: true + editable: true + filterable: true + displayedInFileList: false + - id: "23e45678-e90b-12d3-a456-765114174321" + name: "Comment" + type: "TEXT" + reportingPlaceholder: "{{file.attribute.Comment}}" + displaySettings: + primaryAttribute: false + editable: true + filterable: false + displayedInFileList: false + CsvImportSettings: + type: object + description: | + This object defines the settings for importing data from a CSV file. It includes + information that indicates if the CSV mapping is active, specifies the column header + used for filename matching, sets the delimiter for column separation, and determines + the encoding of the CSV file. These settings are crucial for accurately importing and + mapping file attributes based on the corresponding CSV records. + properties: + csvMappingActive: + type: boolean + description: | + If `true`, the CSV mapping is active. + filenameMappingCsvColumnHeader: + type: string + description: | + The header of a specific column in the CSV file that should contain values + matching the filenames. A matching value identifies the record that is used + to import the mapped file attributes. + delimiter: + type: string + maxLength: 1 + description: | + The delimiter of the CSV file that is used to distinguish different columns. + encoding: + type: string + description: | + The encoding of the CSV file that is expected when uploading for the import of file attributes. + example: + csvMappingActive: true + filenameMappingCsvColumnHeader: "Filename" + delimiter: "," + encoding: "UTF-8" + ReportTemplateIdList: + type: array + items: + type: string + format: uuid + uniqueItems: true + description: | + List of template identifiers indicating which templates are used for generating reports or other outputs. + The reports are generated when requesting a download package. + example: + - b79cb3ba-745e-5d9a-8903-4a02327a7e09 + - fb3463a0-7d6e-54a3-bcd8-1b93388c648d + DossierAttributes: + type: object + description: Additional dossier attributes that can be set + properties: + attributeIdToValue: + additionalProperties: + type: string + type: object + example: + attributeIdToValue: + "1049a73c-8013-45d6-8217-0845a4ff1c61": This is a dossier attribute value + "79d5a138-d30a-4014-ad7f-43ffba1f4d04": This is yet another dossier attribute value + "1d30d9e8-4a6c-4ef0-96a0-7bef62e138db": "1234" + "b337b65a-0481-48d9-92e6-79e34760ef01": "1. January 1337" + Dossier: + type: object + description: | + The `Dossier` object captures all relevant information related to a specific dossier. A dossier is a collection + of documents, managed and analyzed based on settings and rules of the `DossierTemplate` it is derived from. + properties: + id: + type: string + format: uuid + description: | + A unique identifier for the dossier. This ID is automatically generated by + the system upon creation and is used for referencing the dossier in API calls. + name: + type: string + description: | + User-defined name of the dossier, capturing its essence. The name needs to be unique + for the entire workspace (i.e., tenant). + date: + type: string + format: date-time + description: | + The date when the dossier was created. This is automatically set by the system upon the dossier's creation. + dossierTemplateId: + type: string + format: uuid + description: The unique identifier for the template that governs the settings and rules for this dossier. + description: + type: string + description: A detailed narrative explaining the purpose and contents of the dossier. + ownerId: + type: string + format: uuid + description: | + The identifier of the user or entity responsible for the dossier. Typically, + this is the creator or main contact point for the dossier. Only users with "Manager" + privileges can create dossiers and become dossier owners. + memberIds: + type: array + items: + type: string + format: uuid + uniqueItems: true + description: List of unique user identifiers associated with this dossier, such as contributors or viewers. + approverIds: + type: array + items: + type: string + format: uuid + uniqueItems: true + description: | + List of unique user identifiers with elevated permissions. Needed if using an approval workflow. + + The DocuMine application does not have an approval workflow. However, every member will be + elevated to a privileged member to ensure full functionality. + visibility: + type: string + enum: + - PRIVATE + - PUBLIC + description: | + Determines who can see the dossier. Possible values include: + - `PRIVATE`: Visible to dossier members only. + - `PUBLIC`: Visible to all users. + + The system sets the value based on the permissions defined in the application. + startDate: + type: string + format: date-time + description: The starting date for the lifecycle of this dossier. + dueDate: + type: string + format: date-time + description: The end date or deadline for actions related to this dossier. + dossierStatusId: + type: string + format: uuid + description: The unique identifier of the current status of the dossier. The status definitions are managed in the dossier template. + watermarkId: + type: string + format: uuid + deprecated: true + description: | + In DocuMine, watermarks are not supported. + previewWatermarkId: + type: string + format: uuid + deprecated: true + description: | + In DocuMine, watermarks are not supported. + dossierAttributes: + $ref: '#/components/schemas/DossierAttributes' + downloadFileTypes: + $ref: '#/components/schemas/DownloadFileTypes' + reportTemplateIds: + $ref: '#/components/schemas/ReportTemplateIdList' + archivedTime: + type: string + format: date-time + description: The date and time when the dossier was archived. Archived dossiers are read-only and cannot be modified. + softDeletedTime: + type: string + format: date-time + description: | + The timestamp indicating when the dossier was softly deleted, i.e., hidden from + most users but not completely removed from the system. `null` if not deleted. + + A soft-deleted dossier can be restored within a retention period configured in the application. + hardDeletedTime: + type: string + format: date-time + description: The timestamp indicating when the dossier was permanently removed from the system. `null` if not deleted. + example: + id: 2ef05d6a-6fb3-4f27-b3c7-9a6438b38c3b + name: Project Alpha + date: 2000-01-23T04:56:07.000+00:00 + dossierTemplateId: 1ed2db85-9cd5-48b0-855f-77ca2a688501 + description: Collection of documents for Project Alpha. + ownerId: a0044ae9-ddca-4f97-b0a1-3cb2517dbf39 + memberIds: + - da8d5f98-ae61-4696-85bf-27986d93877c + - f4a0beb4-4034-4b92-a75e-5f37591326ea + approverIds: + - da8d5f98-ae61-4696-85bf-27986d93877c + - f4a0beb4-4034-4b92-a75e-5f37591326ea + visibility: PRIVATE + startDate: null + dueDate: 2001-01-23T04:56:07.000+00:00 + dossierStatusId: b8280985-f558-43c0-852a-a3fa86803548 + watermarkId: null + previewWatermarkId: null + downloadFileTypes: + - ORIGINAL + - PREVIEW + reportTemplateIds: + - daadea5f-917b-482a-b7d2-e65afe8f80ca + - 8130acf6-4910-4123-827c-caacd8111402 + archivedTime: null + softDeletedTime: null + hardDeletedTime: null + DossierList: + type: object + description: A list of dossiers. + properties: + dossiers: + items: + $ref: '#/components/schemas/Dossier' + type: array + example: + dossiers: + - id: 2ef05d6a-6fb3-4f27-b3c7-9a6438b38c3b + name: Project Alpha + date: 2000-01-23T04:56:07.000+00:00 + dossierTemplateId: 1ed2db85-9cd5-48b0-855f-77ca2a688501 + description: Collection of documents for Project Alpha. + ownerId: a0044ae9-ddca-4f97-b0a1-3cb2517dbf39 + memberIds: + - da8d5f98-ae61-4696-85bf-27986d93877c + - f4a0beb4-4034-4b92-a75e-5f37591326ea + approverIds: + - da8d5f98-ae61-4696-85bf-27986d93877c + - f4a0beb4-4034-4b92-a75e-5f37591326ea + visibility: PUBLIC + startDate: null + dueDate: 2001-01-23T04:56:07.000+00:00 + dossierStatusId: b8280985-f558-43c0-852a-a3fa86803548 + watermarkId: null + previewWatermarkId: null + downloadFileTypes: + - ORIGINAL + - PREVIEW + reportTemplateIds: + - daadea5f-917b-482a-b7d2-e65afe8f80ca + - 8130acf6-4910-4123-827c-caacd8111402 + archivedTime: null + softDeletedTime: null + hardDeletedTime: null + - id: dc28cc82-2682-4b7f-ae0b-2132b205d47a + name: Project Beta + date: 2001-01-23T04:56:07.000+00:00 + dossierTemplateId: 1ed2db85-9cd5-48b0-855f-77ca2a688501 + description: Collection of documents for Project Beta. + ownerId: 0d53a7c3-7b3a-4c57-857d-ec5e0fc3019b + memberIds: + - a0044ae9-ddca-4f97-b0a1-3cb2517dbf39 + - c2e33246-e50a-4c43-831c-6789a5637db8 + - 6123fa16-6943-4b74-8524-54b0046a0ce6 + visibility: PUBLIC + startDate: null + dueDate: 2001-01-23T04:56:07.000+00:00 + dossierStatusId: b8280985-f558-43c0-852a-a3fa86803548 + watermarkId: null + previewWatermarkId: null + downloadFileTypes: + - ORIGINAL + - PREVIEW + reportTemplateIds: + - daadea5f-917b-482a-b7d2-e65afe8f80ca + - 8130acf6-4910-4123-827c-caacd8111402 + archivedTime: null + softDeletedTime: null + hardDeletedTime: null + DossierRequest: + type: object + description: | + Object containing essential attributes for creating or updating a dossier. + Includes information about ownership, members, approvers, and various configurations + like watermarks and download file types. + properties: + id: + type: string + format: uuid + description: | + The unique identifier of the dossier to update. This property needs to be null for create requests, + as the system will generate it automatically. If the value is not a valid identifier, the + endpoint will return a `400 Bad request` error. + name: + type: string + description: | + A human-readable name for the dossier. The name must be unique among all dossiers + within the scope of the workspace (tenant). + description: + type: string + description: | + A textual description providing additional context or information + about the purpose and content of the dossier. + dueDate: + type: string + format: date-time + description: | + Specifies the deadline by which actions related to this dossier should be completed. + The date is in ISO 8601 date-time format. + ownerId: + type: string + format: uuid + description: | + A unique identifier representing the user who owns or created this dossier. + If `null` when creating the dossier, the current user will be set as the owner. + Must not be `null` when updating a dossier. + memberIds: + type: array + description: An array of unique identifiers for users who have access to the dossier. + items: + type: string + description: A unique identifier for a member with access to the dossier. + uniqueItems: true + reportTemplateIds: + $ref: "#/components/schemas/ReportTemplateIdList" + dossierStatusId: + type: string + description: | + An identifier representing the current status of the dossier. + Use `null` to unset the current status. + required: + - name + example: + id: dc28cc82-2682-4b7f-ae0b-2132b205d47a + name: Project Beta + description: Collection of documents for Project Beta. + dueDate: 2001-01-23T04:56:07.000+00:00 + ownerId: 0d53a7c3-7b3a-4c57-857d-ec5e0fc3019b + memberIds: + - a0044ae9-ddca-4f97-b0a1-3cb2517dbf39 + - c2e33246-e50a-4c43-831c-6789a5637db8 + - 6123fa16-6943-4b74-8524-54b0046a0ce6 + reportTemplateIds: + - daadea5f-917b-482a-b7d2-e65afe8f80ca + - 8130acf6-4910-4123-827c-caacd8111402 + dossierStatusId: b8280985-f558-43c0-852a-a3fa86803548 + ComponentMapping: + description: | + The `ComponentMappingMetadata` object represents the metadata of a component mapping csv file. These CSV files may be used in the component rules to relate components to an existing knowledge base. + type: object + properties: + id: + description: | + A unique identifier for the component mapping metadata. It's generated automatically. Use this to retrieve a specific component mapping file. + type: string + name: + description: | + The name of the component mapping. + type: string + fileName: + description: | + The name of the uploaded component mapping file. + type: string + version: + description: | + The version of the file. It is incremented with each update to the file. + type: integer + columnLabels: + description: | + A list of column labels found in the component mapping file. + type: array + items: + type: string + numberOfLines: + description: | + The number of lines in the component mapping file. + type: integer + format: int32 + encoding: + description: | + The encoding used for the component mapping file. + type: string + delimiter: + description: | + The delimiter used for separating values in the component mapping file. + type: string + required: + - id + - name + - fileName + - columnLabels + - numberOfLines + - encoding + - delimiter + example: + id: 24ff9c3c-4863-4aea-8eda-cab8838b9192 + name: MasterDataMapping + fileName: master_data.csv + columnLabels: + - Column1 + - Column2 + - Column3 + numberOfLines: 100 + encoding: UTF-8 + delimiter: ',' + ComponentMappingSummary: + description: | + The `ComponentMappingSummary` object represents a collection of ComponentMappingMetadata. + type: object + example: + dossierTemplateId: 1e07cde0-d36a-4ab7-b389-494ca694a0cb + componentMappingMetadata: + - id: 24ff9c3c-4863-4aea-8eda-cab8838b9192 + name: MasterDataMapping + fileName: master_data.csv + columnLabels: + - Column1 + - Column2 + - Column3 + numberOfLines: 100 + encoding: UTF-8 + delimiter: ',' + - id: 2e07cde0-d36a-4ab7-b389-494ca694a0cb + name: RegulationNameMapping + fileName: regulation-names.csv + columnLabels: + - Year + - Title + - Identifier + - Name + numberOfLines: 150 + encoding: UTF-16 + delimiter: ; + properties: + dossierTemplateId: + description: | + The identifier of the dossier template associated with the component mapping summaries. + type: string + format: uuid + ComponentMappingMetadata: + description: | + A list of component mapping metadata associated with this dossier template. + type: array + items: + $ref: "#/components/schemas/ComponentMapping" + required: + - dossierTemplateId + - ComponentMappingSummary + DossierTemplate: + description: | + The `DossierTemplate` object represents the blueprint for creating and + managing dossiers and for handling and analyzing the files of dossiers. + It contains various settings, rules, and metadata to control the behavior + of individual dossiers that get created based on this template. + example: + id: 1e07cde0-d36a-4ab7-b389-494ca694a0cb + name: DocuMine Example + description: Typical settings for DocuMine. + dateAdded: 2020-01-23T04:56:07.000+00:00 + dateModified: 2021-01-23T04:56:07.000+00:00 + createdBy: c2e33246-e50a-4c43-831c-6789a5637db6 + modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6 + validFrom: 2020-01-01T00:00:00.000+00:00 + validTo: 2030-12-31T23:59:59.999+00:00 + dossierTemplateStatus: ACTIVE + removeWatermark: false + keepImageMetadata: true + ocrByDefault: true + keepHiddenText: true + keepOverlappingObjects: true + applyDictionaryUpdatesToAllDossiersByDefault: false + downloadFileTypes: + - ORIGINAL + properties: + id: + description: | + A unique identifier generated by the system when the dossier template + is created. This ID is used to reference the template in API calls. + type: string + format: uuid + name: + description: The user-defined name of the dossier template, required for create and update operations. + type: string + description: + description: A detailed description of the purpose and usage of this dossier template. + type: string + dateAdded: + description: The date when this dossier template was first created. Automatically set by the system. + format: date-time + type: string + dateModified: + description: | + The most recent date when modifications were made to this dossier template. + Automatically set by the system. + format: date-time + type: string + createdBy: + description: The identifier of the user who initially created this dossier template. Automatically set by the system. + type: string + format: uuid + modifiedBy: + description: The identifier of the user who last modified this dossier template. Automatically set by the system. + type: string + format: uuid + validFrom: + description: | + The starting date for the validity of this dossier template. Dossiers can only be created + based on this template after this date. + format: date-time + type: string + validTo: + description: | + The ending date for the validity of this dossier template. Dossiers will no longer be able to + be created based on this template after this date. + format: date-time + type: string + downloadFileTypes: + $ref: '#/components/schemas/DownloadFileTypes' + status: + description: | + Indicates the current status of the dossier template: + * `INCOMPLETE`: The dossier template is missing some information or settings. Dossiers cannot be created based + on this template until the missing information and settings are added. + * `INACTIVE`: The dossier template cannot be used for creating dossiers based on it. + * `ACTIVE`: The dossier template can be used for creating dossiers based on it. + enum: + - INCOMPLETE + - INACTIVE + - ACTIVE + type: string + keepImageMetadata: + description: Flag indicating whether metadata from images in PDFs should be retained or removed. + type: boolean + keepHiddenText: + description: Flag indicating whether hidden text in PDFs should be retained or removed. + type: boolean + keepOverlappingObjects: + description: Flag indicating whether overlapping objects in PDFs should be retained or flattened. + type: boolean + applyDictionaryUpdatesToAllDossiersByDefault: + description: | + Flag indicating whether updates to the dictionary should automatically be applied to all dossiers + created from this template. + type: boolean + ocrByDefault: + description: Flag specifying if OCR should be automatically applied to PDFs that get upload to dossiers based on this template. + type: boolean + removeWatermark: + description: Flag specifying if the system should try to remove watermarks in documents prior to OCR processing. + type: boolean + type: object + DownloadFileTypes: + type: array + uniqueItems: true + description: | + Specifies the types of files that will part of the created download package. The defaults can be defined in the dossier template + and can be overwritten individually on each download. + + DocuMine supports `ORIGINAL` and `PREVIEW`: + + - `ORIGINAL` Contrary to intuition, this is not the uploaded file, but the pre-processed, + optimized PDF, which may also contain the OCR results. + This is the PDF that used by the system for further processing. + - `PREVIEW` The annotated version of the PDF, highlighting the found entities that were + evaluated to extract the components. + items: + enum: + - ORIGINAL + - PREVIEW + type: string + example: + - ORIGINAL + DossierTemplateList: + description: Represents a list of dossier templates, each encapsulating a set of rules and settings. + example: + dossierTemplates: + - id: 1e07cde0-d36a-4ab7-b389-494ca694a0cb + name: DocuMine Example + description: Typical settings for DocuMine. + dateAdded: 2020-01-23T04:56:07.000+00:00 + dateModified: 2021-01-23T04:56:07.000+00:00 + createdBy: c2e33246-e50a-4c43-831c-6789a5637db6 + modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6 + validFrom: 2020-01-01T00:00:00.000+00:00 + validTo: 2030-12-31T23:59:59.999+00:00 + dossierTemplateStatus: ACTIVE + removeWatermark: false + keepImageMetadata: true + ocrByDefault: true + keepHiddenText: true + keepOverlappingObjects: true + applyDictionaryUpdatesToAllDossiersByDefault: false + downloadFileTypes: + - ORIGINAL + - id: 8d8cae48-5c33-4617-ac27-1643f29b79d8 + name: Another DocuMine Example + description: Typical settings for DocuMine. + dateAdded: 2023-09-01T06:54:32.000+00:00 + dateModified: 2023-09-01T06:54:32.000+00:00 + createdBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d + modifiedBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d + validFrom: 2023-01-01T00:00:00.000+00:00 + validTo: 2033-12-31T23:59:59.999+00:00 + dossierTemplateStatus: ACTIVE + removeWatermark: true + keepImageMetadata: true + ocrByDefault: true + keepHiddenText: true + keepOverlappingObjects: true + applyDictionaryUpdatesToAllDossiersByDefault: false + downloadFileTypes: + - ORIGINAL + properties: + dossierTemplates: + description: Each entry is a dossier template with its details. + items: + $ref: '#/components/schemas/DossierTemplate' + type: array + type: object + ErrorMessage: + type: object + description: | + Represents an error message returned by the API, providing details on when the error occurred + and a description of the issue. + properties: + timestamp: + type: string + format: date-time + description: The exact date and time when the error was encountered. This can be useful for logging or troubleshooting. + message: + type: string + description: A detailed description of the error, providing insights on the problem and potentially how to resolve or avoid it. + example: + timestamp: "2023-09-21T12:45:00Z" + message: "An error occurred while processing the request." + FileAttributes: + type: object + description: Additional file attributes that can be set or imported + properties: + attributeIdToValue: + additionalProperties: + type: string + type: object + example: + attributeIdToValue: + "9049a73c-8013-45d6-8217-0845a4ff1c61": This is a file attribute value + "59d5a138-d30a-4014-ad7f-43ffba1f4d04": This is yet another file attribute value + "9d30d9e8-4a6c-4ef0-96a0-7bef62e138db": "1234" + "a337b65a-0481-48d9-92e6-79e34760ef01": "1. January 1337" + FileDeleteRequest: + type: object + description: Request payload to initiate the deletion of specific files. + properties: + fileIds: + type: array + description: A list of unique identifiers for the files to be deleted. + items: + type: string + description: The unique identifier of a file. + example: + fileIds: + - 51d3f70ac322c98dc4db70a2ac44115a + - 1fdbd888b39059c8cf171df26f62f8a5 + FileErrorInfo: + type: object + description: Detailed information about an error encountered with a file. + properties: + cause: + type: string + description: The underlying reason or cause of the error. + queue: + type: string + description: The queue or process where the error was encountered. + service: + type: string + description: The specific service or component that reported the error. + timestamp: + type: string + format: date-time + description: The exact time when the error was recorded. + example: + service: LayoutParsingService + cause: The reason or cause why something went wrong. + queue: LAYOUT_PARSING_REQUEST_QUEUE + timestamp: 2000-01-23T04:56:07.000+00:00 + FileStatus: + type: object + description: Object containing information on a specific file. + properties: + id: + type: string + description: The unique identifier of the file. + dossierId: + type: string + format: uuid + description: The unique identifier linking the file to its associated dossier. + dossierTemplateId: + type: string + format: uuid + description: The identifier of the dossier template of the file's dossier. + dossierStatusId: + type: string + format: uuid + description: The identifier of the dossier status of the file's dossier. + filename: + description: The file's name. + type: string + dossierArchived: + description: Tells if the dossier of this file is archived or not. + type: boolean + processingStatus: + type: string + enum: + - ANALYSE + - ERROR + - FULLREPROCESS + - IMAGE_ANALYZING + - INDEXING + - NER_ANALYZING + - OCR_PROCESSING_QUEUED + - OCR_PROCESSING + - PROCESSED + - PROCESSING + - REPROCESS + - UNPROCESSED + - FULL_PROCESSING + - PRE_PROCESSING_QUEUED + - PRE_PROCESSING + - PRE_PROCESSED + - FIGURE_DETECTION_ANALYZING + - TABLE_PARSING_ANALYZING + description: | + The processing status of a file. The states give detailed information about the current processing step. + In general, this information is useful for debugging is the file is stuck in a status. + + The status `UNPROCESSED` indicates a newly uploaded file. Status `ERROR` indicates that the system could not + process the file for some reason. Finally, `PROCESSED` is what you usually expect as the final state after + all required processing steps are done. + workflowStatus: + type: string + enum: + - NEW + - UNDER_REVIEW + - UNDER_APPROVAL + - APPROVED + description: | + The workflow status of a file. + + As DocuMine does not have an approval workflow the meaning of the states is as follows: + - `NEW` - Initial status of the uploaded files. + - `UNDER_REVIEW` is actually not used in DocuMine. + - `UNDER_APPROVAL` means "In progress", i.e. a user is checking the extracted components. + - `APPROVED` means "Done", i.e. a user has checked and adjusted the components if necessary. + numberOfPages: + description: The number of pages of the file. + format: int32 + type: integer + added: + description: Date and time when the file was added to the system. + format: date-time + type: string + lastUpdated: + description: Date and time when the file was last updated. + format: date-time + type: string + numberOfAnalyses: + description: The number of times the file has been analyzed. + format: int32 + type: integer + assignee: + description: The current assignee's (if any) user id. + type: string + lastReviewer: + description: The last reviewer's (if any) user id. + type: string + lastApprover: + description: The user id of the last approver (if any). + type: string + hasRedactions: + description: Shows if any redactions were found during the analysis. + type: boolean + hasHints: + description: Shows if any hints were found during the analysis. + type: boolean + hasRequests: + description: Shows if any requests were found during the analysis. + type: boolean + hasUpdates: + description: | + Shows if there is any change between the previous and current + analysis. + type: boolean + hasImages: + description: Shows if any images were found during the analysis. + type: boolean + ocrStartTime: + description: | + Shows if this file has been OCRed by us. Start time of OCR + Process + format: date-time + type: string + numberOfPagesToOCR: + description: Number of pages to be OCRed by us + format: int32 + type: integer + numberOfOCRedPages: + description: Number of pages already OCRed by us + format: int32 + type: integer + ocrEndTime: + description: Shows if this file has been OCRed by us. End time of OCR Process + format: date-time + type: string + hasAnnotationComments: + description: Shows if this file has comments on annotations. + type: boolean + uploader: + description: The ID of the user who uploaded the file. + type: string + dictionaryVersion: + description: Shows which dictionary versions was used during the analysis. + format: int64 + type: integer + rulesVersion: + description: Shows which rules versions was used during the analysis. + format: int64 + type: integer + legalBasisVersion: + description: Shows which legal basis versions was used during the analysis. + format: int64 + type: integer + excluded: + description: Shows if the file was excluded from analysis. + type: boolean + lastProcessed: + description: Shows the last date of a successful analysis. + format: date-time + type: string + lastLayoutProcessed: + description: Shows the last date of a layout parsing. + format: date-time + type: string + approvalDate: + description: Shows the date of approval, if approved. + format: date-time + type: string + lastUploaded: + description: Shows last date the document was uploaded. + format: date-time + type: string + analysisDuration: + description: Shows how long the last analysis took + format: int64 + type: integer + fileAttributes: + $ref: '#/components/schemas/FileAttributes' + dossierDictionaryVersion: + description: | + Shows which dossier dictionary versions was used during the + analysis. + format: int64 + type: integer + analysisRequired: + description: Shows if the file requires reanalysis. + type: boolean + excludedPages: + description: Set of excluded pages for this file. + items: + description: Set of excluded pages for this file. + format: int32 + type: integer + type: array + uniqueItems: true + softDeletedTime: + description: Shows if the file is soft deleted. + format: date-time + type: string + lastFileAttributeChange: + description: Date and time when the files attributes was last updated. + format: date-time + type: string + hasSuggestions: + description: Shows if there are any Suggestions in this file. + type: boolean + excludedFromAutomaticAnalysis: + description: Shows if the file is excluded from automatic analysis. + type: boolean + redactionModificationDate: + description: Shows the last redaction modification date of this file. + format: date-time + type: string + fileManipulationDate: + description: Shows the date of the last manipulation of this file. + format: date-time + type: string + lastManualChangeDate: + description: | + Shows the date of the last manual change of an annotation of + this file. + format: date-time + type: string + hasHighlights: + description: | + Shows if there are highlights to remove or convert for this + file. + type: boolean + fileSize: + description: Size of the optimized, internally stored file. + format: int64 + type: integer + analysisVersion: + description: Analysis Version. + format: int32 + type: integer + lastIndexed: + description: Last time the file was indexed in ES. + format: date-time + type: string + fileErrorInfo: + $ref: '#/components/schemas/FileErrorInfo' + lastOCRTime: + description: Shows if this file has been OCRed by us. Last Time of OCR. + format: date-time + type: string + example: + hasRedactions: true + added: 2000-01-23T04:56:07.000+00:00 + workflowStatus: NEW + hasAnnotationComments: true + rulesVersion: 2 + excluded: true + lastUpdated: 2000-01-23T04:56:07.000+00:00 + lastManualChangeDate: 2000-01-23T04:56:07.000+00:00 + lastProcessed: 2000-01-23T04:56:07.000+00:00 + excludedFromAutomaticAnalysis: true + dictionaryVersion: 5 + dossierArchived: true + hasRequests: true + lastFileAttributeChange: 2000-01-23T04:56:07.000+00:00 + id: a7f0303d-a33c-4f2e-bd7a-a2adc1b8f92b + analysisDuration: 9 + hasHighlights: true + dossierStatusId: dossierStatusId + fileErrorInfo: + service: service + cause: cause + queue: queue + timestamp: 2000-01-23T04:56:07.000+00:00 + processingStatus: ANALYSE + numberOfAnalyses: 6 + filename: filename + fileAttributes: + myFileAttribute: This is a file attribute value + yetAnotherFileAttribute: This is yet another file attribute value + numericValuesNeedToBeStrings: "1234" + lastOCRTime: 2000-01-23T04:56:07.000+00:00 + legalBasisVersion: 7 + excludedPages: + - 2 + - 2 + redactionModificationDate: 2000-01-23T04:56:07.000+00:00 + fileSize: 4 + fileManipulationDate: 2000-01-23T04:56:07.000+00:00 + dossierId: dossierId + lastUploaded: 2000-01-23T04:56:07.000+00:00 + assignee: assignee + analysisRequired: true + approvalDate: 2000-01-23T04:56:07.000+00:00 + numberOfPagesToOCR: 1 + ocrEndTime: 2000-01-23T04:56:07.000+00:00 + hasHints: true + softDeletedTime: 2000-01-23T04:56:07.000+00:00 + numberOfPages: 0 + uploader: uploader + lastReviewer: lastReviewer + lastIndexed: 2000-01-23T04:56:07.000+00:00 + numberOfOCRedPages: 5 + analysisVersion: 7 + lastLayoutProcessed: 2000-01-23T04:56:07.000+00:00 + lastApprover: lastApprover + ocrStartTime: 2000-01-23T04:56:07.000+00:00 + dossierDictionaryVersion: 3 + dossierTemplateId: dossierTemplateId + hasImages: true + hasUpdates: true + hasSuggestions: true + FileStatusList: + type: object + description: Represents a list detailing the status of multiple files. + properties: + files: + type: array + description: An array of status details for each individual file. + items: + $ref: '#/components/schemas/FileStatus' + example: + files: + - hasRedactions: true + added: 2000-01-23T04:56:07.000+00:00 + workflowStatus: NEW + hasAnnotationComments: true + rulesVersion: 2 + excluded: true + lastUpdated: 2000-01-23T04:56:07.000+00:00 + lastManualChangeDate: 2000-01-23T04:56:07.000+00:00 + lastProcessed: 2000-01-23T04:56:07.000+00:00 + excludedFromAutomaticAnalysis: true + dictionaryVersion: 5 + dossierArchived: true + hasRequests: true + lastFileAttributeChange: 2000-01-23T04:56:07.000+00:00 + id: id + analysisDuration: 9 + hasHighlights: true + dossierStatusId: dossierStatusId + fileErrorInfo: + service: service + cause: cause + queue: queue + timestamp: 2000-01-23T04:56:07.000+00:00 + processingStatus: ANALYSE + numberOfAnalyses: 6 + filename: filename + fileAttributes: + myFileAttribute: This is a file attribute value + yetAnotherFileAttribute: This is yet another file attribute value + numericValuesNeedToBeStrings: "1234" + lastOCRTime: 2000-01-23T04:56:07.000+00:00 + legalBasisVersion: 7 + excludedPages: + - 2 + - 2 + redactionModificationDate: 2000-01-23T04:56:07.000+00:00 + fileSize: 4 + fileManipulationDate: 2000-01-23T04:56:07.000+00:00 + dossierId: 15849e05-5414-498c-b48b-47afa3fd74da + lastUploaded: 2000-01-23T04:56:07.000+00:00 + assignee: assignee + fileId: 4d2334def5fced0003888e47cbc270f7 + analysisRequired: true + approvalDate: 2000-01-23T04:56:07.000+00:00 + numberOfPagesToOCR: 1 + ocrEndTime: 2000-01-23T04:56:07.000+00:00 + hasHints: true + softDeletedTime: 2000-01-23T04:56:07.000+00:00 + numberOfPages: 0 + uploader: uploader + lastReviewer: lastReviewer + lastIndexed: 2000-01-23T04:56:07.000+00:00 + numberOfOCRedPages: 5 + analysisVersion: 7 + lastLayoutProcessed: 2000-01-23T04:56:07.000+00:00 + lastApprover: lastApprover + ocrStartTime: 2000-01-23T04:56:07.000+00:00 + dossierDictionaryVersion: 3 + dossierTemplateId: dossierTemplateId + hasImages: true + hasUpdates: true + hasSuggestions: true + - hasRedactions: true + added: 2000-01-23T04:56:07.000+00:00 + workflowStatus: NEW + hasAnnotationComments: true + rulesVersion: 2 + excluded: true + lastUpdated: 2000-01-23T04:56:07.000+00:00 + lastManualChangeDate: 2000-01-23T04:56:07.000+00:00 + lastProcessed: 2000-01-23T04:56:07.000+00:00 + excludedFromAutomaticAnalysis: true + dictionaryVersion: 5 + dossierArchived: true + hasRequests: true + lastFileAttributeChange: 2000-01-23T04:56:07.000+00:00 + id: id + analysisDuration: 9 + hasHighlights: true + dossierStatusId: dossierStatusId + fileErrorInfo: + service: service + cause: cause + queue: queue + timestamp: 2000-01-23T04:56:07.000+00:00 + processingStatus: ANALYSE + numberOfAnalyses: 6 + filename: filename + fileAttributes: + myFileAttribute: This is a file attribute value + yetAnotherFileAttribute: This is yet another file attribute value + numericValuesNeedToBeStrings: "1234" + lastOCRTime: 2000-01-23T04:56:07.000+00:00 + legalBasisVersion: 7 + excludedPages: + - 2 + - 2 + redactionModificationDate: 2000-01-23T04:56:07.000+00:00 + fileSize: 4 + fileManipulationDate: 2000-01-23T04:56:07.000+00:00 + dossierId: dossierId + lastUploaded: 2000-01-23T04:56:07.000+00:00 + assignee: assignee + fileId: 1fdbd888b39059c8cf171df26f62f8a5 + analysisRequired: true + approvalDate: 2000-01-23T04:56:07.000+00:00 + numberOfPagesToOCR: 1 + ocrEndTime: 2000-01-23T04:56:07.000+00:00 + hasHints: true + softDeletedTime: 2000-01-23T04:56:07.000+00:00 + numberOfPages: 0 + uploader: uploader + lastReviewer: lastReviewer + lastIndexed: 2000-01-23T04:56:07.000+00:00 + numberOfOCRedPages: 5 + analysisVersion: 7 + lastLayoutProcessed: 2000-01-23T04:56:07.000+00:00 + lastApprover: lastApprover + ocrStartTime: 2000-01-23T04:56:07.000+00:00 + dossierDictionaryVersion: 3 + dossierTemplateId: dossierTemplateId + hasImages: true + hasUpdates: true + hasSuggestions: true + FileUploadResult: + description: Object containing information about a successfully uploaded file. + example: + fileIds: + - fileIds + - fileIds + processedFileIds: + - processedFileIds + - processedFileIds + processedAttributes: + - processedAttributes + - processedAttributes + properties: + fileIds: + description: List of fileIds generated for uploaded file(s). + items: + description: List of fileIds generated for uploaded file(s). + type: string + type: array + processedAttributes: + description: | + List processed file attributes, in case the upload contained + a CSV. + items: + description: | + List processed file attributes, in case the upload contained + a CSV. + type: string + type: array + processedFileIds: + description: List processed fileIds, in case the upload contained a CSV. + items: + description: List processed fileIds, in case the upload contained a CSV. + type: string + type: array + type: object + RuleValidationMessage: + description: Object containing information about an uploaded rules file. + example: + line: 123 + column: 45 + message: "Unable to Analyse Expression ..." + properties: + line: + description: The line number where the error or warning occurs. + format: int32 + type: integer + column: + description: The column number where the error or warning occurs. + format: int32 + type: integer + message: + description: The error or warning message that describes the details. + type: string + type: object + RuleValidation: + description: | + Information about the uploaded rules file. The `error` field is empty if there were no validation errors in the uploaded rules file. + example: + errors: + - line: 123 + column: 45 + message: "Unable to Analyse Expression ..." + - line: 234 + column: 5 + message: "Invalid rule syntax ..." + properties: + errors: + description: List of errors found in the uploaded rules file. + items: + $ref: '#/components/schemas/RuleValidationMessage' + type: array + type: object + DownloadStatus: + type: object + description: Detailed information about a specific download. + properties: + id: + type: string + format: uuid + description: The unique identifier of the download. + example: b5e2cf01-8bb6-4fcd-ad88-0efb611195da + userId: + type: string + format: uuid + description: The unique identifier of the user who initiated the download. + example: caa8b54a-eb5e-4134-8ae2-a3946a428ec7 + filename: + type: string + description: The name of the download file. + example: my-component-dossier.zip + mimeType: + type: string + description: The mime type of the download file. + example: application/octet-stream + errorCause: + type: string + description: | + If the status is `FAILED`, this field contains information about the error that happened + while preparing the download package. This information is intended to be included in a + bug report if the error occurs repeatedly and indicates a general problem with DocuMine. + example: "" + status: + type: string + enum: + - QUEUED + - GENERATING + - COMPRESSING + - READY + - FAILED + description: | + The status of the download file. In particular: + - `QUEUED` - The download job has been created and is waiting to be processed by the system. + - `GENERATING` - The system currently creates the files for the download package. + - `COMPRESSING` - The system creates a ZIP archive that will contain all files of the download package. + - `READY` - The download package is ready for download. Please note that the download will be kept only + for a certain period. This period can be configured in the settings of your DocuMine workspace. + - `FAILED` - An error occurred while preparing the download. The `errorCause` field might contain + additional details on the error. + example: READY + creationDate: + type: string + format: date-time + description: The date and time when the user initiated the download. + example: 2023-03-29T11:41:08.886Z + lastDownload: + type: string + format: date-time + description: The date and time when the user last downloaded the file. + example: 2023-03-29T13:11:05.123Z + fileSize: + type: integer + format: int64 + description: The size of the download file in bytes. + example: 1654231 + dossierId: + type: string + format: uuid + description: The identifier of the dossier to which the content of the download package belongs. + example: 20354d7a-e4fe-47af-8ff6-187bca92f3f9 + fileIds: + type: array + items: + type: string + description: The list of file identifiers to which the content of the download package belongs. + example: + - 51d3f70ac322c98dc4db70a2ac44115a + - 1fdbd888b39059c8cf171df26f62f8a5 + downloadFileTypes: + $ref: '#/components/schemas/DownloadFileTypes' + reportTemplateIds: + $ref: '#/components/schemas/ReportTemplateIdList' + DownloadStatusList: + type: object + description: Represents a list detailing the status of multiple downloads. + properties: + downloadStatus: + type: array + items: + $ref: '#/components/schemas/DownloadStatus' + description: Each item contains the status details of a download. + DownloadRequest: + type: object + description: Request payload to initiate the preparation of the download. + properties: + downloadFileTypes: + $ref: '#/components/schemas/DownloadFileTypes' + reportTemplateIds: + $ref: '#/components/schemas/ReportTemplateIdList' + redactionPreviewColor: + type: string + example: "#9398a0" + description: | + A hexadecimal color code that is used to define the highlighting color of the redaction annotations in + the `PREVIEW` file. + + - Black is `#000000` + - White is `#ffffff` + - Grey is `#cccccc` + BulkDownloadRequest: + allOf: + - $ref: '#/components/schemas/DownloadRequest' + - type: object + description: Request payload to initiate the preparation of the download of multiple files. + properties: + fileIds: + type: array + description: A list with unique identifiers of the files for which the download is to be prepared. + items: + type: string + description: The unique identifier of a file. + User: + type: object + description: Basic information about a user. + properties: + id: + type: string + format: uuid + description: The unique identifier of the user. + example: efe7eedd-89c5-56f5-984c-0712ee41a2eb + username: + type: string + description: The user name that is used for logging in. + example: myusername + email: + type: string + description: The email address of the user. + example: myusername@example.com + firstName: + type: string + description: The first name of the user. + example: John + lastName: + type: string + description: The last name of the user. + example: Doe + roles: + uniqueItems: true + type: array + description: | + The roles of the user. In particular: + - `USER` - default user permission to work with DocuMine + - `MANAGER` - additional privileges to create and manage dossiers + - `USER_ADMIN` - administration privileges limited to manage users only + - `ADMIN` - general administration privileges + items: + type: string + enum: + - USER + - MANAGER + - USER_ADMIN + - ADMIN + example: + - MANAGER + - ADMIN + active: + type: boolean + description: Indicator if the user is active or not. Only active users can log in. + UserList: + type: object + description: A list of multiple users. + properties: + downloadStatus: + type: array + items: + $ref: '#/components/schemas/User' + description: Each item contains the details of a user. + LicenseReport: + type: object + description: A comprehensive report of licensing metrics and usage statistics. + properties: + startDate: + type: string + format: date-time + description: The starting date of the report. + endDate: + type: string + format: date-time + description: The ending date of the report. + numberOfAnalyzedFiles: + type: integer + format: int32 + description: | + The count of files that have been analyzed in the requested period. The counter is + increased only once for each file, regardless of the number of analysis. + numberOfAnalyzedPages: + type: integer + format: int32 + description: | + The count of pages that have been analyzed in the requested period. The counter is + increased only once for each file, regardless of the number of analysis. + analyzedFilesBytes: + type: integer + format: int64 + description: | + The size in bytes of the files that have been analyzed in the requested period. + The counter is increased only once for each file, regardless of the number of analysis. + totalFilesUploadedBytes: + type: integer + format: int64 + description: The total size in bytes of all present files at the end of the period. + activeFilesUploadedBytes: + type: integer + format: int64 + description: The size in bytes of all active files at the end of the period. + trashFilesUploadedBytes: + type: integer + format: int64 + description: The size in bytes of all soft-deleted files at the end of the period. + archivedFilesUploadedBytes: + type: integer + format: int64 + description: The size in bytes of all archived files at the end of the period. + numberOfOcrFiles: + type: integer + format: int32 + description: | + The count of files that have undergone OCR (Optical Character Recognition) in the + requested period. + numberOfOcrPages: + type: integer + format: int32 + description: The count of pages that have undergone OCR in the requested period. + numberOfDossiers: + type: integer + format: int32 + description: The total number of present dossiers at the end of the period. + monthlyData: + description: A list of data metrics categorized on a monthly basis. + items: + $ref: '#/components/schemas/MonthlyReportData' + type: array + example: + startDate: 2000-01-01T00:00:00.000+00:00 + endDate: 2001-01-01T00:00:00.000+00:00 + numberOfAnalyzedFiles: 5 + numberOfAnalyzedPages: 9 + analyzedFilesBytes: 30000 + totalFilesUploadedBytes: 30000 + activeFilesUploadedBytes: 20000 + archivedFilesUploadedBytes: 5000 + trashFilesUploadedBytes: 5000 + numberOfOcrFiles: 2 + numberOfOcrPages: 2 + numberOfDossiers: 1 + monthlyData: + - startDate: 2000-01-01T00:00:00.000+00:00 + endDate: 2000-02-01T00:00:00.000+00:00 + numberOfAnalyzedPages: 7 + analyzedFilesBytes: 25000 + totalFilesUploadedBytes: 25000 + activeFilesUploadedBytes: 22000 + archivedFilesUploadedBytes: 0 + trashFilesUploadedBytes: 3000 + numberOfOcrPages: 1 + - startDate: 2000-02-01T00:00:00.000+00:00 + endDate: 2000-03-01T00:00:00.000+00:00 + numberOfAnalyzedPages: 2 + analyzedFilesBytes: 5000 + totalFilesUploadedBytes: 30000 + activeFilesUploadedBytes: 20000 + archivedFilesUploadedBytes: 5000 + trashFilesUploadedBytes: 5000 + numberOfOcrPages: 1 + LicenseReportRequest: + type: object + description: Request object to retrieve a license report for a given date range. + properties: + startDate: + type: string + format: date-time + description: The start date for the requested report. + endDate: + type: string + format: date-time + description: The end date for the requested report. + example: + startDate: 2000-01-01T00:00:00.000+00:00 + endDate: 2001-01-01T00:00:00.000+00:00 + MonthlyReportData: + type: object + description: Detailed metrics regarding license usage for a given month. + properties: + startDate: + type: string + format: date-time + description: The starting date of the respective month. + endDate: + type: string + format: date-time + description: The ending date of the respective month. + numberOfAnalyzedPages: + type: integer + format: int32 + description: | + The count of pages that have been analyzed in the respective month. The counter is + increased only once for each file, regardless of the number of analysis. + analyzedFilesBytes: + type: integer + format: int64 + description: | + The size in bytes of the files that have been analyzed in the respective month. + The counter is increased only once for each file, regardless of the number of analysis. + totalFilesUploadedBytes: + type: integer + format: int64 + description: The total size in bytes of all present files at the end of the respective month. + activeFilesUploadedBytes: + type: integer + format: int64 + description: The size in bytes of all active files at the end of the respective month. + trashFilesUploadedBytes: + type: integer + format: int64 + description: The size in bytes of all soft-deleted files at the end of the respective month. + archivedFilesUploadedBytes: + type: integer + format: int64 + description: The size in bytes of all archived files at the end of the respective month. + numberOfOcrPages: + type: integer + format: int32 + description: The count of pages that have undergone OCR in the requested respective month. + example: + startDate: 2000-01-01T00:00:00.000+00:00 + endDate: 2000-02-01T00:00:00.000+00:00 + numberOfAnalyzedPages: 7 + analyzedFilesBytes: 25000 + totalFilesUploadedBytes: 25000 + activeFilesUploadedBytes: 22000 + archivedFilesUploadedBytes: 0 + trashFilesUploadedBytes: 3000 + numberOfOcrPages: 1 + UploadRequest: + type: object + description: Request object to upload a file. + properties: + file: + type: string + format: binary + description: The binary content of the file to be uploaded. + required: + - file + securitySchemes: + FF-OAUTH: + type: oauth2 + flows: + authorizationCode: + authorizationUrl: /auth/realms/{workspaceId}/protocol/openid-connect/auth + tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token + scopes: { } + clientCredentials: + tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token + scopes: {} \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml similarity index 74% rename from persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml rename to persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml index 4ccd4fb90..9cd097d79 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/openapi.yaml +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml @@ -1,36 +1,37 @@ openapi: 3.0.2 info: - title: DocuMine API - version: "1.0.0" + title: RedactManager API + version: "4.1.0-draft" description: | - The DocuMine API provides a comprehensive solution for managing resources such as dossiers and their associated files. - Users can also retrieve components of files that have been processed and extracted by the system. + The RedactManager API provides a comprehensive solution for managing resources such as dossiers and their associated files. + Users can also retrieve the results package for files that have been processed by the system and reviewed by the users. + The results package can contain the optimized PDF file, the preview PDF, the redacted PDF and correlating redaction reports + in different formats. - Given that DocuMine supports multi-tenancy, it is essential to include the 'X-Tenant-ID' header with every request. - This tenant ID is referred to as the "Workspace ID" within the application. - - All endpoints are secured using OAuth2, with the "authorizationCode" being the supported authorization flow. + All endpoints are secured using OAuth2, with the "authorizationCode" being the general supported authorization flow. Obtain a JWT token for authentication and send it in the 'Authorization' header with the format `Bearer {JWT_TOKEN}`. + + Please also note that the `authorizationUrl` and `tokenUrl` in this specification contain `{workspaceId}` placeholders that + must be replaced by your respective RedactManager workspace identifier. Example Headers: ```properties - X-Tenant-ID: my-workspace Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI... ``` license: name: knecon License contact: name: knecon Service Desk - email: support@documine.ai - url: https://support.documine.ai + email: support@redactmanager.com + url: https://support.redactmanager.com externalDocs: - description: Find out more about DocuMine - url: https://docs.documine.ai + description: Find out more about RedactManager + url: https://docs.redactmanager.com servers: - - url: https://app.documine.ai - description: DocuMine Cloud Service + - url: https://app.redactmanager.com + description: RedactManager Cloud Service security: - - FF-OAUTH: [ ] + - FF-OAUTH: [] tags: - name: 1. Dossier Templates description: Operations related to dossier templates. @@ -39,8 +40,12 @@ tags: - name: 3. Files description: Operations for managing files within a dossier. - name: 4. Components - description: Operations related to components of a file within a dossier. - - name: 5. License + description: Operations related to components of a file within a dossier. These endpoints are not available for RedactManager. + - name: 5. Downloads + description: Operations related to download packages. + - name: 6. Users + description: Operations related to users. + - name: 7. License description: Operations related to license information and usage metrics. paths: /api/dossier-templates: @@ -108,32 +113,25 @@ paths: $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' - /api/dossier-templates/{dossierTemplateId}/entity-rules: + /api/dossier-templates/{dossierTemplateId}/dossier-status-definitions: get: - operationId: downloadEntityRules + summary: Returns the list of all existing dossier status definitions tags: - 1. Dossier Templates - summary: Download the entity rules of a specific dossier template. description: | - Utilize this endpoint to download the entity rules of a designated dossier template. The file is named 'entity-rules.drl' - and contains the set of rules to annotate the entities in an analyzed file. The content of this file is in the Drools Rule Language - (DRL) format. Please find more details about the DRL in the - [Drools Language Reference](https://docs.drools.org/8.44.0.Final/drools-docs/drools/language-reference/index.html). + Retrieves a collection of dossier status definitions associated with a specific dossier template. Each dossier + status definition includes details such as the status name, description, and other relevant metadata. This endpoint + is useful for clients needing to display or set the status of a dossier associated with a specific dossier template. parameters: - $ref: '#/components/parameters/dossierTemplateId' responses: "200": - headers: - Content-Disposition: - schema: - type: string - example: attachment; filename*=utf-8''entity-rules.drl content: - text/plain; charset=utf-8: + '*/*': schema: - type: string + $ref: '#/components/schemas/DossierStatusDefinitionList' description: | - Successfully downloaded the requested rules file. + Successfully returned the dossier status definitions for the specified dossier template. "400": $ref: '#/components/responses/400' "401": @@ -146,75 +144,26 @@ paths: $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' - post: - operationId: uploadEntityRules - tags: - - 1. Dossier Templates - summary: Upload or validate a entity rules file for a specific dossier template. - description: | - Utilize this endpoint to upload the entity rules to a designated dossier template. With the 'dryRun' parameter, - you have the possibility to just validate the rules file without actually saving it in the dossier template. - - The uploaded rule file will be saved only if 'dryRun' is set to `false` and the response code is `200`. In this - case, the response object does not contain errors. - parameters: - - $ref: '#/components/parameters/dossierTemplateId' - - $ref: '#/components/parameters/dryRun' - requestBody: - content: - multipart/form-data: - schema: - $ref: '#/components/schemas/UploadRequest' - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/RuleValidation' - example: - errors: [ ] - description: | - Successfully uploaded or validated the entity rules file. The returned response object will not contain any errors. - "400": - $ref: '#/components/responses/400' - "401": - $ref: '#/components/responses/401' - "403": - $ref: '#/components/responses/403' - "404": - $ref: '#/components/responses/404-dossier' - "422": - $ref: '#/components/responses/422-rules' - "429": - $ref: '#/components/responses/429' - "500": - $ref: '#/components/responses/500' - /api/dossier-templates/{dossierTemplateId}/component-rules: + /api/dossier-templates/{dossierTemplateId}/dossier-attribute-definitions: get: - operationId: downloadComponentRules + summary: Returns the list of all existing dossier attribute definitions tags: - 1. Dossier Templates - summary: Download the component rules of a specific dossier template. description: | - Utilize this endpoint to download the component rules of a designated dossier template. The file is named 'component-rules.drl' - and contains the set of rules to build components based on entities of an analyzed file. The content of this file is in the Drools Rule Language - (DRL) format. Please find more details about the DRL in the - [Drools Language Reference](https://docs.drools.org/8.44.0.Final/drools-docs/drools/language-reference/index.html). + Retrieves a collection of dossier attribute definitions associated with a specific dossier template. Each dossier + attribute definition includes details such as attribute type, name, and other relevant metadata. This endpoint + is useful for clients needing to understand what attributes are expected or allowed for dossiers associated with + a specific dossier template. parameters: - $ref: '#/components/parameters/dossierTemplateId' responses: "200": - headers: - Content-Disposition: - schema: - type: string - example: attachment; filename*=utf-8''component-rules.drl content: - text/plain; charset=utf-8: + '*/*': schema: - type: string + $ref: '#/components/schemas/DossierAttributeDefinitionList' description: | - Successfully downloaded the requested rules file. + Successfully returned the dossier attribute definitions for the specified dossier template. "400": $ref: '#/components/responses/400' "401": @@ -227,49 +176,6 @@ paths: $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' - post: - operationId: uploadComponentRules - tags: - - 1. Dossier Templates - summary: Upload or validate a component rules file for a specific dossier template. - description: | - Utilize this endpoint to upload the component rules to a designated dossier template. With the 'dryRun' parameter, - you have the possibility to just validate the rules file without actually saving it in the dossier template. - - The uploaded rule file will be saved only if 'dryRun' is set to `false` and the response code is `200`. In this - case, the response object does not contain errors. - parameters: - - $ref: '#/components/parameters/dossierTemplateId' - - $ref: '#/components/parameters/dryRun' - requestBody: - content: - multipart/form-data: - schema: - $ref: '#/components/schemas/UploadRequest' - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/RuleValidation' - example: - errors: [ ] - description: | - Successfully uploaded or validated the component rules file. The returned response object will not contain any errors. - "400": - $ref: '#/components/responses/400' - "401": - $ref: '#/components/responses/401' - "403": - $ref: '#/components/responses/403' - "404": - $ref: '#/components/responses/404-dossier' - "422": - $ref: '#/components/responses/422-rules' - "429": - $ref: '#/components/responses/429' - "500": - $ref: '#/components/responses/500' /api/dossier-templates/{dossierTemplateId}/file-attribute-definitions: get: summary: Returns the list of all existing file attribute definitions @@ -278,8 +184,8 @@ paths: description: | Retrieves a collection of file attribute definitions associated with a specific dossier template. Each file attribute definition includes details such as attribute type, name, and other relevant metadata. This endpoint - is useful for clients needing to understand what attributes are expected or allowed for files under a particular - dossier template. + is useful for clients needing to understand what attributes are expected or allowed for files associated with + a specific dossier template. parameters: - $ref: '#/components/parameters/dossierTemplateId' responses: @@ -290,6 +196,18 @@ paths: $ref: '#/components/schemas/FileAttributeDefinitionList' description: | Successfully returned the file attribute definitions for the specified dossier template. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier-template' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' /api/dossier-templates/{dossierTemplateId}/dossiers: get: operationId: getDossiers @@ -448,6 +366,85 @@ paths: $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/attributes: + post: + operationId: setDossierAttributes + tags: + - 2. Dossiers + summary: Update or set attributes for a specific dossier. + description: | + This endpoint facilitates the updating or setting of specific dossier attributes for a given dossier. + Ensure you provide the necessary dossier attributes within the request body. + + Use this route to maintain or enhance dossier metadata and properties. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DossierAttributes' + required: true + responses: + "204": + description: | + Dossier attributes successfully updated. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/create-download: + post: + operationId: prepareDossierDownload + tags: + - 2. Dossiers + summary: Initiate the creation of a download package for all files of a dossier. + description: | + To download the results of a redaction, a download package needs to be prepared. This endpoint + facilitates to define the content of the download package and to start the creation of it for all + files of the dossier. The response of this endpoint contains an identifier for the download that + is needed to query the status until the download package is prepared, and finally to actually + download the file once it is available. + + Note: The redacted file will be created for `APPROVED` files only. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadStatus' + description: | + Successfully started the creation of the download package for all files of the requested dossier. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files: get: operationId: getDossierStatus @@ -568,10 +565,14 @@ paths: File deletion successful. This confirms the absence of the specified file, irrespective of its previous existence. "400": $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' "403": $ref: '#/components/responses/403' "404": $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' get: @@ -598,10 +599,58 @@ paths: Successfully retrieved the status of the requested file. "400": $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' "403": $ref: '#/components/responses/403' "404": $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/{fileId}/create-download: + post: + operationId: prepareFileDownload + tags: + - 3. Files + summary: Initiate the creation of a download package for a single file of a dossier. + description: | + To download the results of a redaction, a download package needs to be prepared. This endpoint + facilitates to define the content of the download package and to start the creation of it for the + file. The response of this endpoint contains an identifier for the download that is needed to + query the status until the download package is prepared for download, and finally to actually + download the file once it is available. + + Note: The redacted PDF will be created for `APPROVED` files only. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + - $ref: '#/components/parameters/fileId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadStatus' + description: | + Successfully started the creation of the download package for the requested file. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/bulk/delete: @@ -634,10 +683,57 @@ paths: Bulk file deletion successful. "400": $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' "403": $ref: '#/components/responses/403' "404": $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/bulk/create-download: + post: + operationId: prepareBulkDownload + tags: + - 3. Files + summary: Initiate the creation of a download package for specific files within a dossier. + description: | + To download the results of a redaction, a download package needs to be prepared. This endpoint + facilitates to define the content of the download package and to start the creation of it for + multiple files of a dossier. The response of this endpoint contains an identifier for the download + that is needed to query the status until the download package is prepared for download, and finally + to actually download the file once it is available. + + Note: The redacted PDF will be created for `APPROVED` files only. + parameters: + - $ref: '#/components/parameters/dossierTemplateId' + - $ref: '#/components/parameters/dossierId' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BulkDownloadRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadStatus' + description: | + Successfully started the creation of the download package for the requested files. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-dossier' + "429": + $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/{fileId}/attributes: @@ -662,94 +758,204 @@ paths: $ref: '#/components/schemas/FileAttributes' required: true responses: - "200": + "204": description: | File attributes successfully updated. "400": $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' "403": $ref: '#/components/responses/403' "404": $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' - /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/{fileId}/components: + /api/download: get: - operationId: getComponents + operationId: getDownloadStatusList tags: - - 4. Components - summary: Fetch the components associated with a specific file. + - 5. Downloads + summary: Get the list of downloads for the current user description: | - This endpoint retrieves the derived and extracted components from a given file within a dossier. Components represent - various aspects, metadata or content of the file. Entity and component rules define these components based on the file's - content. They can give a *structured view* on a document's text. - - To include detailed component information, set the `includeDetails` query parameter to `true`. - parameters: - - $ref: '#/components/parameters/dossierTemplateId' - - $ref: '#/components/parameters/dossierId' - - $ref: '#/components/parameters/fileId' - - $ref: '#/components/parameters/includeComponentDetails' + This endpoint facilitates to retrieve the list of downloads for the current user. The response contains + status objects that represent each download and provide information on whether the download is still + in preparation, is already available or whether an error occurred when the download package was created. responses: "200": content: application/json: schema: - $ref: '#/components/schemas/FileComponents' - application/xml: - schema: - $ref: '#/components/schemas/FileComponents' - description: | - Successfully retrieved file components. + $ref: '#/components/schemas/DownloadStatusList' + description: List of all downloads of the current users successfully retrieved. "400": $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' "403": $ref: '#/components/responses/403' - "404": - $ref: '#/components/responses/404-file' + "429": + $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' - /api/dossier-templates/{dossierTemplateId}/dossiers/{dossierId}/files/bulk/get-components: + /api/downloads/{downloadId}: get: - operationId: getComponentsOfDossier + operationId: getDownloadStatus tags: - - 4. Components - summary: Returns the RSS response for all files of a dossier + - 5. Downloads + summary: Get the status for a specific download of the current user description: | - This endpoint fetches components for all files associated with a specified dossier. Like individual file components, - these represent various aspects, metadata or content of the files. Entity and component rules define these components based on the file's - content. They can give a *structured view* on a document's text. - - To include detailed component information, set the `includeDetails` query parameter to `true`. + This endpoint facilitates to retrieve the status for a specific download. In addition to other + information, the status indicates whether the download is still being prepared, is already + available, or whether an error occurred when the download package was created. parameters: - - $ref: '#/components/parameters/dossierTemplateId' - - $ref: '#/components/parameters/dossierId' - - $ref: '#/components/parameters/fileId' - - $ref: '#/components/parameters/includeComponentDetails' + - $ref: '#/components/parameters/downloadId' responses: "200": content: application/json: schema: - $ref: '#/components/schemas/FileComponentsList' - application/xml: - schema: - $ref: '#/components/schemas/FileComponentsList' - description: | - Successfully fetched components for all files in the dossier. - "404": - $ref: '#/components/responses/404-dossier' + $ref: '#/components/schemas/DownloadStatus' + description: Status of the download successfully retrieved. "400": $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' "403": $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-download' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + delete: + operationId: deleteDownload + tags: + - 5. Downloads + summary: Deletes a specific download. + description: | + This endpoint facilitates the removal of a download. + parameters: + - $ref: '#/components/parameters/downloadId' + responses: + "204": + description: | + Download deletion successful. This confirms the absence of the specified download, irrespective of its previous existence. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-download' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/downloads/{downloadId}/download: + get: + operationId: download + tags: + - 5. Downloads + - persistence-service + summary: Download the download package. + description: | + This endpoint facilitates to actually download the created download package. The request will + only be successful if the status of the download is `READY`. + parameters: + - $ref: '#/components/parameters/downloadId' + responses: + "200": + headers: + Content-Disposition: + schema: + type: string + example: attachment; filename*=utf-8''example.zip + content: + application/octet-stream: + schema: + type: string + format: binary + description: Successfully downloaded the requested file. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-download' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/users: + get: + operationId: getUsers + tags: + - 6. Users + - tenant-user-management + summary: Get a list of users + description: | + This endpoint facilitates to retrieve a list of known users. + + With the `username` parameter you can filter for a specific user name. If the parameter is + used, the returned list either contains a single matching entry or is empty. + parameters: + - $ref: '#/components/parameters/username' + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/UserList' + description: List of users successfully retrieved. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "429": + $ref: '#/components/responses/429' + "500": + $ref: '#/components/responses/500' + /api/users/{userId}: + get: + operationId: getUserById + tags: + - 6. Users + summary: Retrieve a specific user by its identifier. + description: | + This endpoint facilitates to retrieve a specific user. + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/User' + description: User successfully retrieved. + "400": + $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' + "403": + $ref: '#/components/responses/403' + "404": + $ref: '#/components/responses/404-user' + "429": + $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' /api/license/active/usage: post: - operationId: getReport + operationId: getLicenseReport tags: - - 5. License + - 7. License summary: Generate and retrieve a license usage report. description: | This endpoint enables users to create and fetch a report detailing the active usage of licenses. The report contains @@ -771,18 +977,16 @@ paths: License report successfully generated and retrieved. "400": $ref: '#/components/responses/400' + "401": + $ref: '#/components/responses/401' "403": $ref: '#/components/responses/403' + "429": + $ref: '#/components/responses/429' "500": $ref: '#/components/responses/500' components: headers: - X-Tenant-ID: - description: Tenant identifier, also known as the *Workspace ID* in the application. - required: true - schema: - type: string - example: 'my-workspace' Authorization: description: JWT token for authorization. Format should be `Bearer {JWT_TOKEN}`. required: true @@ -839,6 +1043,20 @@ components: Some endpoints support a `includeSoftDeleted` parameter: If this is set to `true`, this response is returned only if the file is deleted permanently. + "404-download": + content: + '*/*': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + Download not found. This happens if the requested download does not exist for the current user. + "404-user": + content: + '*/*': + schema: + $ref: '#/components/schemas/ErrorMessage' + description: | + User not found. This happens if the requested user does not exist. "409-dossier-conflict": content: '*/*': @@ -846,13 +1064,6 @@ components: $ref: '#/components/schemas/ErrorMessage' description: | Name conflict: The provided name is already in use by another dossier. It needs to be unique in the scope of your workspace. - 422-rules: - content: - '*/*': - schema: - $ref: '#/components/schemas/RuleValidation' - description: | - Invalid rules file: There were validation errors, the rules file is unprocessable. "429": content: '*/*': @@ -1082,20 +1293,26 @@ components: overwriting a file: - `true`: The system keeps the manual changes on re-uploading (overwriting) the file. - `false` (default): Manual changes get discarded on re-uploading the file. - includeComponentDetails: - name: includeDetails + downloadId: + name: downloadId + in: path + required: true + schema: + type: string + style: simple + explode: false + description: The identifier for the file to download. + username: + name: username in: query required: false schema: - default: false - type: boolean + type: string style: form explode: true description: | - A toggle to decide whether to include detailed component information in the response: - - `true`: The component object's field `componentDetails` stores detailed information about the *source* of its - respective value(s). - - `false` (default): The component object does not contain a field `componentDetails`. + If the `username` parameter is set, the user list is filtered for that specific user name. This means the list + either has one matching entry or is empty. schemas: EntityReference: type: object @@ -1128,273 +1345,96 @@ components: entityRuleId: ABC.0.0 type: entity_type page: 123 - Component: + DossierStatusDefinition: type: object - description: Represents a component with its associated values. + description: | + The `DossierStatusDefinition` object contains the relevant information to define a dossier status. The dossier status + is used to assign a custom status to a dossier. properties: + id: + type: string + format: uuid + description: | + A unique identifier for the dossier status definition. This ID is automatically generated by + the system upon creation and is used for referencing the dossier status definition in API calls. + example: bcd22239-cedf-442f-a5a1-1664cba94dc6 name: type: string - description: The name of the component. - componentValues: - type: array - description: A list of value objects associated with the component. - items: - $ref: '#/components/schemas/ComponentValue' - example: - name: the component name - componentValues: - - value: my overwritten component value - originalValue: the original value - componentRuleId: COM.0.0 - valueDescription: My value description - entityReferences: - - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 - entityRuleId: ABC.0.0 - type: entity_type - page: 123 - - id: bcd22239-c3df-442f-a5a1-1664cba94dc6_2 - entityRuleId: ABC.0.0 - type: entity_type - page: 124 - - id: b748b89a-5679-4254-9286-1dd652d9970b - entityRuleId: DEF.13.37 - type: another_entity_type - page: 456 - - value: yet another component value - originalValue: yet another component value - componentRuleId: COM.0.1 - valueDescription: Another value description - entityReferences: - - id: 70496456-a016-4679-81b1-6c8856dded6e - entityRuleId: XYZ.0.0 - type: yet_another_entity_type - page: 123 - ComponentValue: + description: | + User-defined name of the dossier status definition, capturing its essence. The name needs to be unique + for the dossier template. + example: "Done" + description: + type: string + description: | + A text that can be added to provide further details about the status. E.g., what it is intended for or + the circumstances under which it can be used. + example: "Dossiers with this status should only contain approved files and indicate that the users have completed the redactions." + rank: + format: int32 + type: integer + description: | + A number that allows to define a custom display order. + default: "" + example: 1 + color: + type: string + description: | + A hexadecimal color code that can be set to assign a color to a + the `PREVIEW` file. + + - Yellow is `#ffda05` + - Green is `#5eb160` + example: "#5eb160" + required: + - name + - rank + - color + DossierAttributeDefinition: type: object - description: Describes a value associated with a component, including its origin and related entities found in the document. + description: | + The `DossierAttributeDefinition` object contains the relevant information to define a dossier attribute. Dossier attributes + are used to manage additional meta-data of dossiers. properties: - value: + id: type: string - description: The current value of the component. - originalValue: - type: string - description: The original value before any modifications. - componentRuleId: - type: string - description: Identifier for the rule associated with this component value. - valueDescription: - type: string - description: A brief description of the value. - entityReferences: - items: - $ref: '#/components/schemas/EntityReference' - type: array - example: - value: my overwritten component value - originalValue: the original value - componentRuleId: COM.0.0 - valueDescription: My value description - entityReferences: - - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 - entityRuleId: ABC.0.0 - type: entity_type - page: 123 - - id: b748b89a-5679-4254-9286-1dd652d9970b - entityRuleId: DEF.13.37 - type: another_entity_type - page: 456 - FileComponents: - type: object - description: Represents file details along with its associated components and values. - properties: - dossierTemplateId: format: uuid + description: | + A unique identifier for the dossier attribute definition. This ID is automatically generated by + the system upon creation and is used for referencing the dossier attribute definition in API calls. + name: type: string - description: Identifier for the template associated with the dossier. - dossierId: - format: uuid + description: | + User-defined name of the dossier attribute definition, capturing its essence. The name needs to be unique + for the dossier template. + type: type: string - description: Identifier for the dossier. - fileId: + enum: + - TEXT + - NUMBER + - DATE + description: | + Determines the type of the dossier attribute's value. Please note that currently the system + does not validate the values against this definition. This is just a hint for a user interface + that needs to handle invalid entries. Possible values for the type: + - `TEXT`: The value is just a string, i.e., any sequence of characters. + - `NUMBER`: The value is a string expressing a number, with or without decimals. + - `DATE`: The value is a string expressing a date information. + reportingPlaceholder: type: string - description: Identifier for the file. - filename: - type: string - description: Name of the file. - components: - type: object - description: A map of component names to their list of values. - additionalProperties: - items: - type: string - type: array - componentDetails: - type: object - description: A map of component names to their detailed representations. - additionalProperties: - $ref: '#/components/schemas/Component' + description: | + The name of the placeholder of the dossier attribute that can be used in report templates. The + placeholder follows a specific format convention: + `{{dossier.attribute.}}` while the name is transformed into 'PascalCase' and does not contain + whitespaces. The placeholder is unique in a dossier template. + required: + - name + - type example: - dossierTemplateId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 - dossierId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 - fileId: 1fdbd888b39059c8cf171df26f62f8a5 - filename: MyFile.pdf - components: - "my component": - - my overwritten component value - - yet another component value - "yet another component": - - only one value - componentDetails: - - name: my component - componentValues: - - value: my overwritten component value - originalValue: the original value - componentRuleId: COM.0.0 - valueDescription: My value description - entityReferences: - - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 - entityRuleId: ABC.0.0 - type: entity_type - page: 123 - - id: b748b89a-5679-4254-9286-1dd652d9970b - entityRuleId: DEF.13.37 - type: another_entity_type - page: 456 - - value: yet another component value - originalValue: yet another component value - componentRuleId: COM.0.1 - valueDescription: Another value description - entityReferences: - - id: 70496456-a016-4679-81b1-6c8856dded6e - entityRuleId: XYZ.0.0 - type: yet_another_entity_type - page: 123 - - name: yet another component - componentValues: - - value: only one value - originalValue: only one value - componentRuleId: COM.0.0 - valueDescription: My value description - entityReferences: - - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 - entityRuleId: ABC.0.0 - type: entity_type - page: 123 - - id: b748b89a-5679-4254-9286-1dd652d9970b - entityRuleId: DEF.13.37 - type: another_entity_type - page: 456 - FileComponentsList: - type: object - description: A list of files and their associated components. - properties: - files: - type: array - description: List of files with their component details. - items: - $ref: '#/components/schemas/FileComponents' - example: - files: - - dossierTemplateId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 - dossierId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 - fileId: 1fdbd888b39059c8cf171df26f62f8a5 - filename: MyFile.pdf - components: - "my component": - - my overwritten component value - - yet another component value - "yet another component": - - only one value - componentDetails: - - name: my component - componentValues: - - value: my overwritten component value - originalValue: the original value - componentRuleId: COM.0.0 - valueDescription: My value description - entityReferences: - - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 - entityRuleId: ABC.0.0 - type: entity_type - page: 123 - - id: b748b89a-5679-4254-9286-1dd652d9970b - entityRuleId: DEF.13.37 - type: another_entity_type - page: 456 - - value: yet another component value - originalValue: yet another component value - componentRuleId: COM.0.1 - valueDescription: Another value description - entityReferences: - - id: 70496456-a016-4679-81b1-6c8856dded6e - entityRuleId: XYZ.0.0 - type: yet_another_entity_type - page: 123 - - name: yet another component - componentValues: - - value: only one value - originalValue: only one value - componentRuleId: COM.0.0 - valueDescription: My value description - entityReferences: - - id: bcd22239-cedf-442f-a5a1-1664cba94dc6 - entityRuleId: ABC.0.0 - type: entity_type - page: 123 - - id: b748b89a-5679-4254-9286-1dd652d9970b - entityRuleId: DEF.13.37 - type: another_entity_type - page: 456 - - dossierTemplateId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 - dossierId: 046b6c7f-0b8a-43b9-b35d-6489e6daee91 - fileId: 4d2334def5fced0003888e47cbc270f7 - filename: Copy of MyFile.pdf - components: - "my component": - - my overwritten component value - - yet another component value - "yet another component": - - only one value - componentDetails: - - name: my component - componentValues: - - value: my overwritten component value - originalValue: the original value - componentRuleId: COM.0.0 - valueDescription: My value description - entityReferences: - - id: dadb54d4-587b-4e69-8c07-be9446a33537 - entityRuleId: ABC.0.0 - type: entity_type - page: 123 - - id: 30339f5a-4c19-447f-b0a0-0f14c094037e - entityRuleId: DEF.13.37 - type: another_entity_type - page: 456 - - value: yet another component value - originalValue: yet another component value - componentRuleId: COM.0.1 - valueDescription: Another value description - entityReferences: - - id: 086cd500-97da-44f8-8628-3324d11b4e8d - entityRuleId: XYZ.0.0 - type: yet_another_entity_type - page: 123 - - name: yet another component - componentValues: - - value: only one value - originalValue: only one value - componentRuleId: COM.0.0 - valueDescription: My value description - entityReferences: - - id: dadb54d4-587b-4e69-8c07-be9446a33537 - entityRuleId: ABC.0.0 - type: entity_type - page: 123 - - id: 30339f5a-4c19-447f-b0a0-0f14c094037e - entityRuleId: DEF.13.37 - type: another_entity_type - page: 456 + id: "123e4567-e89b-12d3-a456-426614174000" + name: "Document Summary" + type: "TEXT" + reportingPlaceholder: "{{dossier.attribute.DocumentSummary}}" FileAttributeDefinition: type: object description: | @@ -1419,7 +1459,7 @@ components: - NUMBER - DATE description: | - Determines the type of the dossier attribute's value. Please note that currently the system + Determines the type of the file attribute's value. Please note that currently the system does not validate the values against this definition. This is just a hint for a user interface that needs to handle invalid entries. Possible values for the type: - `TEXT`: The value is just a string, i.e., any sequence of characters. @@ -1433,7 +1473,7 @@ components: type: string description: | The name of the placeholder of the file attribute that can be used in report templates. The placeholder follows a specific format convention: - `{{file.attribute.}}` while the name transformed into 'PascalCase' and does not contain whitespaces. The placeholder is unique in a dossier template. + `{{file.attribute.}}` while the name is transformed into 'PascalCase' and does not contain whitespaces. The placeholder is unique in a dossier template. displaySettings: $ref: '#/components/schemas/FileAttributeDisplaySettings' required: @@ -1453,24 +1493,24 @@ components: FileAttributeDisplaySettings: type: object description: | - Display setting for the RedactManager and DocuMine user interface. These settings control how the UI handles and presents the file attributes. + Display setting for the RedactManager user interface. These settings control how the UI handles and presents the file attributes. properties: primaryAttribute: type: boolean description: | - If `true`, the RedactManager and DocuMine user interfaces show the value of the file attribute in the file list below the file name. + If `true`, the RedactManager user interfaces show the value of the file attribute in the file list below the file name. editable: type: boolean description: | - If `true`, the RedactManager and DocuMine user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible. + If `true`, the RedactManager user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible. filterable: type: boolean description: | - If `true`, the RedactManager and DocuMine user interfaces add filter options to the file list. + If `true`, the RedactManager user interfaces add filter options to the file list. displayedInFileList: type: boolean description: | - if `true`, the RedactManager and DocuMine user interfaces show the values in the file list. + if `true`, the RedactManager user interfaces show the values in the file list. required: - primaryAttribute - editable @@ -1481,6 +1521,44 @@ components: editable: true filterable: true displayedInFileList: false + DossierStatusDefinitionList: + type: object + description: A list of dossier status definitions. + properties: + dossierStatusDefinitions: + items: + $ref: '#/components/schemas/DossierStatusDefinition' + type: array + example: + dossierStatusDefinitions: + - id: "123e7567-e89b-12d3-a456-426614174000" + name: "In Progress" + description: "Dossiers with this status are currently being processed by the users." + rank: 0 + color: "#ffda05" + - id: "23e45378-e90b-12d3-a456-765114174321" + name: "Done" + description: "Dossiers with this status should only contain approved files and indicate that the users have completed the redactions." + rank: 1 + color: "#5eb160" + DossierAttributeDefinitionList: + type: object + description: A list of dossier attribute definitions. + properties: + dossierAttributeDefinitions: + items: + $ref: '#/components/schemas/DossierAttributeDefinition' + type: array + example: + dossierAttributeDefinitions: + - id: "123e4567-e89b-12d3-a456-426614174000" + name: "Dossier Summary" + type: "TEXT" + reportingPlaceholder: "{{dossier.attribute.DossierSummary}}" + - id: "23e45678-e90b-12d3-a456-765114174321" + name: "Comment" + type: "TEXT" + reportingPlaceholder: "{{dossier.attribute.Comment}}" FileAttributeDefinitionList: type: object description: A list of file attribute definitions. @@ -1550,6 +1628,32 @@ components: filenameMappingCsvColumnHeader: "Filename" delimiter: "," encoding: "UTF-8" + ReportTemplateIdList: + type: array + items: + type: string + format: uuid + uniqueItems: true + description: | + List of template identifiers indicating which templates are used for generating reports or other outputs. + The reports are generated when requesting a download package. + example: + - b79cb3ba-745e-5d9a-8903-4a02327a7e09 + - fb3463a0-7d6e-54a3-bcd8-1b93388c648d + DossierAttributes: + type: object + description: Additional dossier attributes that can be set + properties: + attributeIdToValue: + additionalProperties: + type: string + type: object + example: + attributeIdToValue: + "1049a73c-8013-45d6-8217-0845a4ff1c61": This is a dossier attribute value + "79d5a138-d30a-4014-ad7f-43ffba1f4d04": This is yet another dossier attribute value + "1d30d9e8-4a6c-4ef0-96a0-7bef62e138db": "1234" + "b337b65a-0481-48d9-92e6-79e34760ef01": "1. January 1337" Dossier: type: object description: | @@ -1601,9 +1705,6 @@ components: uniqueItems: true description: | List of unique user identifiers with elevated permissions. Needed if using an approval workflow. - - The DocuMine application does not have an approval workflow. However, every member will be - elevated to a privileged member to ensure full functionality. visibility: type: string enum: @@ -1633,41 +1734,18 @@ components: deprecated: true description: | Identifier for the watermark that's to be applied on redacted documents within this dossier. - - In DocuMine, watermarks are not supported. previewWatermarkId: type: string format: uuid deprecated: true description: | Identifier for the watermark pattern used for generated previews documents within this dossier. - - In DocuMine, watermarks are not supported. + dossierAttributes: + $ref: '#/components/schemas/DossierAttributes' downloadFileTypes: - type: array - items: - enum: - - ORIGINAL - - PREVIEW - - REDACTED - - ANNOTATED - - FLATTEN - - DELTA_PREVIEW - type: string - uniqueItems: true - description: | - Types of files available for download from the dossier. These types can - differ based on the application. DocuMine only supports `ORIGINAL`. The files are - provided or generated when requesting a download package. + $ref: '#/components/schemas/DownloadFileTypes' reportTemplateIds: - type: array - items: - type: string - format: uuid - uniqueItems: true - description: | - List of template identifiers indicating which templates are to be used for generating reports or outputs - for this dossier. The reports are generated when requesting a download package. + $ref: '#/components/schemas/ReportTemplateIdList' archivedTime: type: string format: date-time @@ -1819,14 +1897,7 @@ components: description: A unique identifier for a member with access to the dossier. uniqueItems: true reportTemplateIds: - type: array - description: | - An array of identifiers representing templates used for generating reports - or exports from this dossier. - items: - description: An identifier for a report template. - type: string - uniqueItems: true + $ref: "#/components/schemas/ReportTemplateIdList" dossierStatusId: type: string description: | @@ -1918,30 +1989,7 @@ components: format: date-time type: string downloadFileTypes: - description: | - Specifies the types of files that will be set as default types to download for dossiers created from - this template. Valid options may vary depending on the system. - - * Valid options for RedactManager: - * `ORIGINAL`: The optimized version of the PDF like it is used by the system for further analysis. - * `PREVIEW`: The optimized PDF with redaction annotations indicating what gets redacted by the - system. Note that the content to redact is actually still present and readable. - * `REDACTED`: The redacted PDF - * `DELTA_PREVIEW`: If redactions were imported (e.g., by uploading a PDF with redaction annotations), - this PDF highlights the changes made to imported redactions in different colors (green: no change, - red: removed, blue: added). - * Valid options for DocuMine: - * `ORIGINAL`: The optimized version of the PDF like it is used by the system for further analysis. - items: - description: Enumerated type indicating a permissible download file type for dossiers under this template. - enum: - - ORIGINAL - - PREVIEW - - REDACTED - - DELTA_PREVIEW - type: string - type: array - uniqueItems: true + $ref: '#/components/schemas/DownloadFileTypes' status: description: | Indicates the current status of the dossier template: @@ -1975,6 +2023,38 @@ components: description: Flag specifying if the system should try to remove watermarks in documents prior to OCR processing. type: boolean type: object + DownloadFileTypes: + type: array + uniqueItems: true + description: | + Specifies the types of files that will part of the created download package. The defaults can be defined in the dossier template + and can be overwritten individually on each download. + + RedactManager supports `ORIGINAL`, `PREVIEW`, `DELTA_PREVIEW`, and `REDACTED`: + + - `ORIGINAL` Contrary to intuition, this is not the uploaded file, but the pre-processed, + optimized PDF, which may also contain the OCR results. + This is the PDF that used by the system for further processing. + - `PREVIEW` The annotated version of the PDF indicating what gets redacted by the + system. Note that the content to redact is actually still present and readable. + The redaction information is embedded so you can restore the redactions when uploading the + `PREVIEW` PDF into RedactManager. + - `DELTA_PREVIEW` Shows changes if redactions were imported (e.g., by uploading a Preview PDF + with redaction annotations): + Unchanged imported redactions are highlighted in green, removed imported redactions are + highlighted in red, changed imported redactions (e.g. resized) are highlighted in yellow, + and additional redactions are highlighted in blue. + - `REDACTED` The ISO27038 compliant sanitized PDF file that contains all redactions. + items: + enum: + - ORIGINAL + - PREVIEW + - DELTA_PREVIEW + - REDACTED + type: string + example: + - PREVIEW + - REDACTED DossierTemplateList: description: Represents a list of dossier templates, each encapsulating a set of rules and settings. example: @@ -1999,8 +2079,8 @@ components: - PREVIEW - REDACTED - id: 8d8cae48-5c33-4617-ac27-1643f29b79d8 - name: DocuMine Example - description: Typical settings for DocuMine. + name: Another RedactManager Example + description: Typical settings for RedactManager. dateAdded: 2023-09-01T06:54:32.000+00:00 dateModified: 2023-09-01T06:54:32.000+00:00 createdBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d @@ -2009,13 +2089,15 @@ components: validTo: 2033-12-31T23:59:59.999+00:00 dossierTemplateStatus: ACTIVE removeWatermark: true - keepImageMetadata: true + keepImageMetadata: false ocrByDefault: true keepHiddenText: true - keepOverlappingObjects: true + keepOverlappingObjects: false applyDictionaryUpdatesToAllDossiersByDefault: false downloadFileTypes: - ORIGINAL + - PREVIEW + - REDACTED properties: dossierTemplates: description: Each entry is a dossier template with its details. @@ -2048,9 +2130,11 @@ components: type: string type: object example: - myFileAttribute: This is a file attribute value - yetAnotherFileAttribute: This is yet another file attribute value - numericValuesNeedToBeStrings: "1234" + attributeIdToValue: + "9049a73c-8013-45d6-8217-0845a4ff1c61": This is a file attribute value + "59d5a138-d30a-4014-ad7f-43ffba1f4d04": This is yet another file attribute value + "9d30d9e8-4a6c-4ef0-96a0-7bef62e138db": "1234" + "a337b65a-0481-48d9-92e6-79e34760ef01": "1. January 1337" FileDeleteRequest: type: object description: Request payload to initiate the deletion of specific files. @@ -2150,11 +2234,15 @@ components: description: | The workflow status of a file. - As DocuMine does not have an approval workflow the meaning of the states is as follows: - - `NEW` means "New", nothing else. - - `UNDER_REVIEW` is actually not used in DocuMine. - - `UNDER_APPROVAL` means "In progress", i.e. a user is checking the extracted components. - - `APPROVED` means "Done", i.e. a user has checked and adjusted the components if necessary. + - `NEW` - Initial status of the uploaded files. + - `UNDER_REVIEW` - The redactions of the document are under review. In this status, users can make corrections + and add redactions. + - `UNDER_APPROVAL` - The redactions have been reviewed and the document is ready for final approval. With this additional status, + a four-eyes-principle can be applied. However, users can still make final corrections and add redactions before + approving the file. + - `APPROVED` - Files have been reviewed and finally approved. Redactions of files in this final state are no longer modified by the + system's automatic analysis and are also read-only for users. I.e., the files must be returned to a previous status to make + changes to the redactions. numberOfPages: description: The number of pages of the file. format: int32 @@ -2390,7 +2478,7 @@ components: hasSuggestions: true FileStatusList: type: object - description: Represents a list detailing the status of a bunch of files. + description: Represents a list detailing the status of multiple files. properties: files: type: array @@ -2559,43 +2647,180 @@ components: type: string type: array type: object - RuleValidationMessage: - description: Object containing information about an uploaded rules file. - example: - line: 123 - column: 45 - message: "Unable to Analyse Expression ..." + DownloadStatus: + type: object + description: Detailed information about a specific download. properties: - line: - description: The line number where the error or warning occurs. - format: int32 - type: integer - column: - description: The column number where the error or warning occurs. - format: int32 - type: integer - message: - description: The error or warning message that describes the details. + id: type: string - type: object - RuleValidation: - description: | - Information about the uploaded rules file. The `error` field is empty if there were no validation errors in the uploaded rules file. - example: - errors: - - line: 123 - column: 45 - message: "Unable to Analyse Expression ..." - - line: 234 - column: 5 - message: "Invalid rule syntax ..." - properties: - errors: - description: List of errors found in the uploaded rules file. - items: - $ref: '#/components/schemas/RuleValidationMessage' + format: uuid + description: The unique identifier of the download. + example: b5e2cf01-8bb6-4fcd-ad88-0efb611195da + userId: + type: string + format: uuid + description: The unique identifier of the user who initiated the download. + example: caa8b54a-eb5e-4134-8ae2-a3946a428ec7 + filename: + type: string + description: The name of the download file. + example: my-redacted-dossier.zip + mimeType: + type: string + description: The mime type of the download file. + example: application/octet-stream + errorCause: + type: string + description: | + If the status is `FAILED`, this field contains information about the error that happened + while preparing the download package. This information is intended to be included in a + bug report if the error occurs repeatedly and indicates a general problem with RedactManager. + example: "" + status: + type: string + enum: + - QUEUED + - GENERATING + - COMPRESSING + - READY + - FAILED + description: | + The status of the download file. In particular: + - `QUEUED` - The download job has been created and is waiting to be processed by the system. + - `GENERATING` - The system currently creates the files for the download package. + - `COMPRESSING` - The system creates a ZIP archive that will contain all files of the download package. + - `READY` - The download package is ready for download. Please note that the download will be kept only + for a certain period. This period can be configured in the settings of your RedactManager workspace. + - `FAILED` - An error occurred while preparing the download. The `errorCause` field might contain + additional details on the error. + example: READY + creationDate: + type: string + format: date-time + description: The date and time when the user initiated the download. + example: 2023-03-29T11:41:08.886Z + lastDownload: + type: string + format: date-time + description: The date and time when the user last downloaded the file. + example: 2023-03-29T13:11:05.123Z + fileSize: + type: integer + format: int64 + description: The size of the download file in bytes. + example: 1654231 + dossierId: + type: string + format: uuid + description: The identifier of the dossier to which the content of the download package belongs. + example: 20354d7a-e4fe-47af-8ff6-187bca92f3f9 + fileIds: type: array + items: + type: string + description: The list of file identifiers to which the content of the download package belongs. + example: + - 51d3f70ac322c98dc4db70a2ac44115a + - 1fdbd888b39059c8cf171df26f62f8a5 + downloadFileTypes: + $ref: '#/components/schemas/DownloadFileTypes' + reportTemplateIds: + $ref: '#/components/schemas/ReportTemplateIdList' + DownloadStatusList: type: object + description: Represents a list detailing the status of multiple downloads. + properties: + downloadStatus: + type: array + items: + $ref: '#/components/schemas/DownloadStatus' + description: Each item contains the status details of a download. + DownloadRequest: + type: object + description: Request payload to initiate the preparation of the download. + properties: + downloadFileTypes: + $ref: '#/components/schemas/DownloadFileTypes' + reportTemplateIds: + $ref: '#/components/schemas/ReportTemplateIdList' + redactionPreviewColor: + type: string + example: "#9398a0" + description: | + A hexadecimal color code that is used to define the highlighting color of the redaction annotations in + the `PREVIEW` file. + + - Black is `#000000` + - White is `#ffffff` + - Grey is `#cccccc` + BulkDownloadRequest: + allOf: + - $ref: '#/components/schemas/DownloadRequest' + - type: object + description: Request payload to initiate the preparation of the download of multiple files. + properties: + fileIds: + type: array + description: A list with unique identifiers of the files for which the download is to be prepared. + items: + type: string + description: The unique identifier of a file. + User: + type: object + description: Basic information about a user. + properties: + id: + type: string + format: uuid + description: The unique identifier of the user. + example: efe7eedd-89c5-56f5-984c-0712ee41a2eb + username: + type: string + description: The user name that is used for logging in. + example: myusername + email: + type: string + description: The email address of the user. + example: myusername@example.com + firstName: + type: string + description: The first name of the user. + example: John + lastName: + type: string + description: The last name of the user. + example: Doe + roles: + uniqueItems: true + type: array + description: | + The roles of the user. In particular: + - `USER` - default user permission to work with RedactManager + - `MANAGER` - additional privileges to create and manage dossiers + - `USER_ADMIN` - administration privileges limited to manage users only + - `ADMIN` - general administration privileges + items: + type: string + enum: + - USER + - MANAGER + - USER_ADMIN + - ADMIN + example: + - MANAGER + - ADMIN + active: + type: boolean + description: Indicator if the user is active or not. Only active users can log in. + UserList: + type: object + description: A list of multiple users. + properties: + downloadStatus: + type: array + items: + $ref: '#/components/schemas/User' + description: Each item contains the details of a user. LicenseReport: type: object description: A comprehensive report of licensing metrics and usage statistics. @@ -2774,9 +2999,12 @@ components: - file securitySchemes: FF-OAUTH: + type: oauth2 flows: authorizationCode: - authorizationUrl: /auth/realms/redaction/protocol/openid-connect/auth - scopes: { } - tokenUrl: /auth/realms/redaction/protocol/openid-connect/token - type: oauth2 + authorizationUrl: /auth/realms/{workspaceId}/protocol/openid-connect/auth + tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token + scopes: {} + clientCredentials: + tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token + scopes: {} \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java index 7d530b5d5..846312fc8 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java @@ -53,6 +53,7 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP public void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AnalyzeResult analyzeResult) { + log.info("Received analysis result {}", analyzeResult); fileStatusProcessingUpdateService.analysisSuccessful(dossierId, fileId, analyzeResult); } diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index dbf220411..6ec669f08 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -61,6 +61,7 @@ dependencies { api("com.opencsv:opencsv:5.4") api("org.springframework.cloud:spring-cloud-starter-openfeign:${springCloudVersion}") api("commons-validator:commons-validator:1.7") + api("com.opencsv:opencsv:5.9") implementation("org.mapstruct:mapstruct:1.5.5.Final") annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java new file mode 100644 index 000000000..67f27f504 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java @@ -0,0 +1,78 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; + +import org.hibernate.annotations.Fetch; +import org.springframework.data.annotation.LastModifiedDate; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; + +import jakarta.persistence.CollectionTable; +import jakarta.persistence.Column; +import jakarta.persistence.ElementCollection; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.Id; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +@Data +@Entity +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Table(name = "component_mappings") +public class ComponentMappingEntity { + + @Id + @Column(name = "id") + String id; + + @NonNull + @ManyToOne(fetch = FetchType.LAZY) + private DossierTemplateEntity dossierTemplate; + + @NonNull + @Builder.Default + String storageId = ""; + + @NonNull + @Builder.Default + String name = ""; + + @NonNull + @Builder.Default + String fileName = ""; + + @NonNull + @Builder.Default + Integer version = -1; + + @LastModifiedDate + OffsetDateTime changedDate; + + @NonNull + @Builder.Default + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name = "component_mapping_column_labels") + List columnLabels = new ArrayList<>(); + + @NonNull + @Builder.Default + Integer numberOfLines = 0; + + @NonNull + @Builder.Default + String encoding = "UTF-8"; + + @Builder.Default + char delimiter = ','; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java index fc3d0f1a3..030752a2b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java @@ -69,8 +69,8 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation { @Column private OffsetDateTime softDeletedTime; - @ElementCollection(fetch = FetchType.EAGER) @Fetch(FetchMode.SUBSELECT) + @ElementCollection(fetch = FetchType.EAGER) private List positions = new ArrayList<>(); @ManyToOne(fetch = FetchType.LAZY) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 61fda011a..4ba0806b4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -14,8 +14,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; import jakarta.persistence.CascadeType; +import jakarta.persistence.CollectionTable; import jakarta.persistence.Column; import jakarta.persistence.Convert; +import jakarta.persistence.ElementCollection; import jakarta.persistence.Entity; import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; @@ -197,6 +199,9 @@ public class FileEntity { @Column private OffsetDateTime errorTimestamp; + @ElementCollection + private List componentMappingVersions; + public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java new file mode 100644 index 000000000..f50700668 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.dossier; + +import jakarta.persistence.Embeddable; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Embeddable +@AllArgsConstructor +@NoArgsConstructor +public class FileEntityComponentMappingVersionEntity { + + private String name; + private Integer version; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java new file mode 100644 index 000000000..7fa8ff3be --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.persistence.management.v1.processor.mapper; + +import java.util.List; + +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; + +@Mapper +public interface ComponentMappingEntityMapper { + + ComponentMappingEntityMapper INSTANCE = Mappers.getMapper(ComponentMappingEntityMapper.class); + + + ComponentMappingMetadata toComponentMappingMetaData(ComponentMappingEntity componentMappingMetaData); + + + List toComponentMappingMetaDataList(List componentMappingEntities); + + + ComponentMappingEntity toComponentMappingEntity(ComponentMappingMetadata componentMappingSummary); + + + List toComponentMappingEntityList(List componentMappingSummary); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMapping.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMapping.java new file mode 100644 index 000000000..c1528c29d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMapping.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.persistence.management.v1.processor.model; + +import java.io.File; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; + +public record ComponentMapping(ComponentMappingMetadata metaData, File file) { + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMappingDownloadModel.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMappingDownloadModel.java new file mode 100644 index 000000000..399c9d76c --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentMappingDownloadModel.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.persistence.management.v1.processor.model; + +import org.springframework.core.io.InputStreamResource; + +public record ComponentMappingDownloadModel(InputStreamResource mappingFileResource, String encoding, String fileName) { + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java new file mode 100644 index 000000000..43a75883b --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java @@ -0,0 +1,104 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.util.List; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; +import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ComponentMappingRepository; +import com.iqser.red.storage.commons.service.StorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; + +@Service +@Transactional +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class ComponentMappingPersistenceService { + + StorageService storageService; + + ComponentMappingRepository repository; + + + public List getByDossierTemplateId(String dossierTemplateId) { + + return repository.findByDossierTemplateId(dossierTemplateId); + } + + + public void deleteById(String dossierTemplateId, String id) { + + ComponentMappingEntity entity = getEntityById(dossierTemplateId, id); + delete(entity); + } + + + public ComponentMappingEntity getEntityById(String dossierTemplateId, String id) { + + return repository.findByIdAndDossierTemplateId(id, dossierTemplateId) + .orElseThrow(() -> new NotFoundException("ComponentMapping with id " + id + " not found!")); + } + + + private void delete(ComponentMappingEntity entity) { + + repository.delete(entity); + storageService.deleteObject(TenantContext.getTenantId(), entity.getStorageId()); + } + + + public void deleteByDossierTemplateId(String dossierTemplateId) { + + repository.findByDossierTemplateId(dossierTemplateId) + .forEach(this::delete); + } + + + @SneakyThrows + public void updateOrCreate(String storageId, File mappingFile, ComponentMappingEntity entity) { + + entity.setChangedDate(OffsetDateTime.now()); + repository.saveAndFlush(entity); + try (var in = new FileInputStream(mappingFile)) { + storageService.storeObject(TenantContext.getTenantId(), storageId, in); + } + } + + + public boolean existsByNameAndDossierTemplateId(String name, String dossierTemplateId) { + + return repository.existsByNameAndDossierTemplateId(name, dossierTemplateId); + } + + + @SneakyThrows + public ComponentMappingDownloadModel getMappingFileForDownload(String dossierTemplateId, String componentMappingId) { + + var entity = getEntityById(dossierTemplateId, componentMappingId); + if (!storageService.objectExists(TenantContext.getTenantId(), entity.getStorageId())) { + throw new NotFoundException("ComponentMapping with id " + componentMappingId + " does not exist!"); + } + return new ComponentMappingDownloadModel(storageService.getObject(TenantContext.getTenantId(), entity.getStorageId()), entity.getEncoding(), entity.getFileName()); + } + + + public File downloadMappingFileToFolder(String storageId, String fileName, Path outputDir) { + + File outputFile = outputDir.resolve(fileName).toFile(); + storageService.downloadTo(TenantContext.getTenantId(), storageId, outputFile); + return outputFile; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java new file mode 100644 index 000000000..cfd7c3483 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java @@ -0,0 +1,225 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.nio.charset.Charset; +import java.nio.charset.IllegalCharsetNameException; +import java.nio.charset.UnsupportedCharsetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.mapper.ComponentMappingEntityMapper; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.CSVWriter; +import com.opencsv.exceptions.CsvException; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; + +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class ComponentMappingService { + + ComponentMappingPersistenceService componentMappingPersistenceService; + DossierTemplatePersistenceService dossierTemplatePersistenceService; + ComponentMappingEntityMapper mappingEntityMapper = ComponentMappingEntityMapper.INSTANCE; + + static CSVSorter CSV_SORTER = new CSVSorter(); + + + public List getMetaDataByDossierTemplateId(String dossierTemplateId) { + + List entities = componentMappingPersistenceService.getByDossierTemplateId(dossierTemplateId); + return mappingEntityMapper.toComponentMappingMetaDataList(entities); + } + + + public ComponentMappingMetadata getMetaData(String dossierTemplateId, String mappingId) { + + return mappingEntityMapper.toComponentMappingMetaData(componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId)); + } + + + @SneakyThrows + public ComponentMappingMetadata update(String dossierTemplateId, String mappingId, String encoding, char delimiter, File mappingFile) { + + ComponentMappingEntity entity = componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId); + + return updateOrCreate(entity, encoding, delimiter, mappingFile); + } + + + @SneakyThrows + public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile) { + + if (componentMappingPersistenceService.existsByNameAndDossierTemplateId(name, dossierTemplateId)) { + throw new BadRequestException("A mapping with this name already exists in the dossier template!"); + } + + String id = UUID.randomUUID().toString(); + String storageId = buildStorageId(dossierTemplateId, id, name, fileName); + ComponentMappingEntity entity = ComponentMappingEntity.builder() + .id(id) + .dossierTemplate(dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId)) + .storageId(storageId) + .name(name) + .fileName(fileName) + .build(); + + return updateOrCreate(entity, encoding, delimiter, mappingFile); + } + + + @SneakyThrows + private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity, String encoding, char delimiter, File mappingFile) { + + Charset charset = resolveCharset(encoding); + + CsvStats stats = sortCSVFile(delimiter, mappingFile, charset); + + entity.setDelimiter(delimiter); + entity.setEncoding(encoding); + entity.setNumberOfLines(stats.numberOfLines()); + entity.setColumnLabels(stats.columnLabels()); + entity.setVersion(entity.getVersion() + 1); + + componentMappingPersistenceService.updateOrCreate(entity.getStorageId(), mappingFile, entity); + + return mappingEntityMapper.toComponentMappingMetaData(entity); + } + + + private static Charset resolveCharset(String encoding) { + + try { + return Charset.forName(encoding); + } catch (IllegalCharsetNameException e) { + throw new BadRequestException("Invalid character encoding: " + encoding); + } catch (UnsupportedCharsetException e) { + throw new BadRequestException("Unsupported character encoding: " + encoding); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Encoding can't be null."); + } + } + + + private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws CsvException, BadRequestException, IOException { + + Path tempFile = Files.createTempFile("mapping", ".tmp"); + + Files.move(mappingFile.toPath(), tempFile, StandardCopyOption.REPLACE_EXISTING); + + String[] columnLabels; + int numberOfLines = 0; + try (Reader fileReader = new FileReader(tempFile.toFile(), charset);// + CSVReader reader = buildReader(fileReader, delimiter);// + CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset))) { + + List rows = reader.readAll(); + + columnLabels = rows.remove(0); + + numberOfLines = (int) reader.getLinesRead() - 1; + + rows.sort(CSV_SORTER); + + writer.writeNext(columnLabels); + + writer.writeAll(rows); + + } catch (IOException e) { + throw new BadRequestException("Error while sorting the csv file", e); + } + + Files.deleteIfExists(tempFile); + + return new CsvStats(Arrays.asList(columnLabels), numberOfLines); + } + + + private static CSVReader buildReader(Reader reader, char delimiter) throws IOException { + + return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build(); + } + + + public void delete(String dossierTemplateId, String componentMappingId) { + + componentMappingPersistenceService.deleteById(dossierTemplateId, componentMappingId); + } + + + public ComponentMappingDownloadModel getMappingForDownload(String dossierTemplateId, String componentMappingId) { + + return componentMappingPersistenceService.getMappingFileForDownload(dossierTemplateId, componentMappingId); + } + + + @SneakyThrows + public List getMappingFilesByDossierTemplateId(String dossierTemplateId, Path outputDir) { + + var entities = componentMappingPersistenceService.getByDossierTemplateId(dossierTemplateId); + + return entities.stream() + .map(mappingEntityMapper::toComponentMappingMetaData) + .map(metaData -> downloadFileAndCreateMapping(outputDir, metaData)) + .toList(); + + } + + + private ComponentMapping downloadFileAndCreateMapping(Path outputDir, ComponentMappingMetadata metaData) { + + File mappingFile = componentMappingPersistenceService.downloadMappingFileToFolder(metaData.getStorageId(), metaData.getFileName(), outputDir); + return new ComponentMapping(metaData, mappingFile); + } + + + public static String buildStorageId(String dossierTemplateId, String id, String name, String fileName) { + + return dossierTemplateId + "/" + id + "_" + name + "_" + fileName; + } + + + private static class CSVSorter implements Comparator { + + @Override + public int compare(String[] line1, String[] line2) { + + for (int column = 0; column < line1.length; column++) { + if (!line1[column].equals(line2[column])) { + return line1[column].compareTo(line2[column]); + } + } + + return 0; + } + + } + + private record CsvStats(List columnLabels, int numberOfLines) { + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java index e9bc1c8cc..f09b35fb7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -10,6 +12,7 @@ import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; +import org.springframework.util.FileSystemUtils; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity; @@ -20,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService; @@ -40,29 +44,33 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; import com.knecon.fforesight.tenantcommons.TenantContext; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DossierTemplateCloneService { - private final DossierTemplateRepository dossierTemplateRepository; - private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; - private final RulesPersistenceService rulesPersistenceService; - private final DossierTemplatePersistenceService dossierTemplatePersistenceService; - private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService; - private final DictionaryPersistenceService dictionaryPersistenceService; - private final EntryPersistenceService entryPersistenceService; - private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; - private final ReportTemplatePersistenceService reportTemplatePersistenceService; - private final ColorsService colorsService; - private final StorageService storageService; - private final DossierStatusPersistenceService dossierStatusPersistenceService; - private final WatermarkService watermarkService; - private final FileManagementStorageService fileManagementStorageService; + DossierTemplateRepository dossierTemplateRepository; + LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + RulesPersistenceService rulesPersistenceService; + DossierTemplatePersistenceService dossierTemplatePersistenceService; + DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService; + DictionaryPersistenceService dictionaryPersistenceService; + EntryPersistenceService entryPersistenceService; + FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; + ReportTemplatePersistenceService reportTemplatePersistenceService; + ColorsService colorsService; + StorageService storageService; + DossierStatusPersistenceService dossierStatusPersistenceService; + WatermarkService watermarkService; + FileManagementStorageService fileManagementStorageService; + ComponentMappingService componentMappingService; public DossierTemplateEntity cloneDossierTemplate(String dossierTemplateId, CloneDossierTemplateRequest cloneDossierTemplateRequest) { @@ -129,6 +137,8 @@ public class DossierTemplateCloneService { // set the watermark configurations cloneWatermarks(dossierTemplateId, clonedDossierTemplate.getId()); + cloneComponentMappings(dossierTemplateId, clonedDossierTemplate.getId()); + return clonedDossierTemplate; } @@ -153,6 +163,23 @@ public class DossierTemplateCloneService { } + @SneakyThrows + private void cloneComponentMappings(String dossierTemplateId, String clonedDossierTemplateId) { + + Path dir = Files.createTempDirectory("componentMappingsForClone"); + List componentMappings = componentMappingService.getMappingFilesByDossierTemplateId(dossierTemplateId, dir); + for (ComponentMapping componentMapping : componentMappings) { + componentMappingService.create(clonedDossierTemplateId, + componentMapping.metaData().getName(), + componentMapping.metaData().getFileName(), + componentMapping.metaData().getDelimiter(), + componentMapping.metaData().getEncoding(), + componentMapping.file()); + } + FileSystemUtils.deleteRecursively(dir); + } + + private void cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) { var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java index fb0c6d290..012946acd 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java @@ -8,14 +8,17 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -60,6 +63,7 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa import com.iqser.red.service.persistence.management.v1.processor.utils.FileUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus; @@ -70,6 +74,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ComponentMappingImportModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ExportFilename; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportTemplateResult; @@ -81,6 +86,7 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -109,6 +115,7 @@ public class DossierTemplateImportService { private final StorageService storageService; private final ObjectMapper objectMapper = new ObjectMapper(); private final FileManagementServiceSettings settings; + private final ComponentMappingService componentMappingService; public String importDossierTemplate(ImportDossierTemplateRequest request) { @@ -145,7 +152,8 @@ public class DossierTemplateImportService { Map> typeEntriesMap = new HashMap<>(); Map> typeFalsePositivesMap = new HashMap<>(); Map> typeFalseRecommendationsMap = new HashMap<>(); - + Map mappingDataMap = new HashMap<>(); + List mappingMetadataList = new LinkedList<>(); while ((ze = zis.getNextZipEntry()) != null) { log.debug("---> " + ze.getName() + " ---- " + ze.isDirectory()); totalEntryArchive++; @@ -274,12 +282,22 @@ public class DossierTemplateImportService { reportTemplateFilenameList = reportTemplateList.stream() .map(rt -> rt.isMultiFileReport() ? rt.getFileName() + ExportFilename.REPORT_TEMPLATE_MULTI_FILE.getFilename() : rt.getFileName()) .toList(); + } else if (ze.getName().startsWith(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename())) { + if (ze.getName().contains(ExportFilename.COMPONENT_MAPPINGS_FILE.getFilename())) { + mappingMetadataList = objectMapper.readValue(bytes, new TypeReference<>() { + }); + } else if (ze.getName().endsWith(".csv")) { + String fileName = ze.getName().replace(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename() + "/", ""); + mappingDataMap.put(fileName, bytes); + } + } else { reportTemplateBytesMap.put(ze.getName(), bos); } bos.close(); } } + importTemplateResult.setEntries(typeEntriesMap); importTemplateResult.setFalsePositives(typeFalsePositivesMap); importTemplateResult.setFalseRecommendations(typeFalseRecommendationsMap); @@ -298,6 +316,13 @@ public class DossierTemplateImportService { } } + + for (var metadata : mappingMetadataList) { + String fileName = metadata.getName() + ".csv"; + if (mappingDataMap.containsKey(fileName)) { + importTemplateResult.componentMappings.add(new ComponentMappingImportModel(metadata, mappingDataMap.get(fileName))); + } + } if (importTemplateResult.getDossierTemplate() == null) { throw new BadRequestException("Provided archive is faulty"); } @@ -335,9 +360,6 @@ public class DossierTemplateImportService { updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId()); dossierTemplateRepository.save(existingDossierTemplate); - // set rules - setRulesWhenCompiled(request, dossierTemplateId); - existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(existingDossierTemplate) .name())); @@ -448,12 +470,11 @@ public class DossierTemplateImportService { dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); dossierTemplateEntity.setCreatedBy(request.getUserId()); //set rules - setRulesWhenCompiled(request, dossierTemplateEntity.getId()); var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity); loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate)); - dossierTemplateId = loadedDossierTemplate.getId(); + dossierTemplateId = loadedDossierTemplate.getId(); // set colors this.setColors(dossierTemplateId, request.getColors()); @@ -527,17 +548,44 @@ public class DossierTemplateImportService { FileAttributesGeneralConfigurationEntity.class)); } + setRulesWhenCompiled(request, dossierTemplateId); + setComponentMappings(dossierTemplateId, request.getComponentMappings()); + long elapsedTime = System.currentTimeMillis() - start; log.info("stop import dossier template elapsedTime: " + elapsedTime + "for: " + dossierTemplateId); return dossierTemplateId; } + @SneakyThrows + private void setComponentMappings(String dossierTemplateId, List componentMappings) { + + List existingMappings = componentMappingService.getMetaDataByDossierTemplateId(dossierTemplateId); + existingMappings.forEach(metadata -> componentMappingService.delete(dossierTemplateId, metadata.getId())); + + for (ComponentMappingImportModel componentMapping : componentMappings) { + File tmpFile = Files.createTempFile("mapping", ".csv").toFile(); + try (var out = new FileOutputStream(tmpFile)) { + out.write(componentMapping.csvData()); + } + componentMappingService.create(dossierTemplateId, + componentMapping.metadata().getName(), + componentMapping.metadata().getFileName(), + componentMapping.metadata().getDelimiter(), + componentMapping.metadata().getEncoding(), + tmpFile); + assert tmpFile.delete(); + } + + } + + private void setRulesWhenCompiled(ImportTemplateResult request, String dossierTemplateId) { DroolsValidation droolsValidation = rulesValidationService.validateRules(RuleFileType.ENTITY, request.getRuleSet()); if (!droolsValidation.isCompiled()) { - droolsValidation.getSyntaxErrorMessages().forEach(errorMessage -> log.error(errorMessage.getMessage())); + droolsValidation.getSyntaxErrorMessages() + .forEach(errorMessage -> log.error(errorMessage.getMessage())); droolsValidation.getBlacklistErrorMessages() .forEach(errorMessage -> log.error(errorMessage.getMessage())); throw new BadRequestException("The entity rules do not compile or contain blacklisted keywords!"); @@ -546,7 +594,8 @@ public class DossierTemplateImportService { if (request.getComponentRuleSet() != null) { DroolsValidation componentDroolsValidation = rulesValidationService.validateRules(RuleFileType.COMPONENT, request.getComponentRuleSet()); if (!componentDroolsValidation.isCompiled()) { - componentDroolsValidation.getSyntaxErrorMessages().forEach(errorMessage -> log.error(errorMessage.getMessage())); + componentDroolsValidation.getSyntaxErrorMessages() + .forEach(errorMessage -> log.error(errorMessage.getMessage())); componentDroolsValidation.getBlacklistErrorMessages() .forEach(errorMessage -> log.error(errorMessage.getMessage())); throw new BadRequestException("The component rules do not compile or contain blacklisted keywords!"); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java index a60f18cd5..ba3903fa3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java @@ -89,7 +89,7 @@ public class DownloadService { request.getReportTemplateIds(), request.getRedactionPreviewColor()); websocketService.sendDownloadEvent(storageId, request.getUserId(), DownloadStatusValue.QUEUED); - addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).includeUnprocessed(request.getIncludeUnprocessed()).build(), 1); + addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).build(), 1); return new JSONPrimitive<>(storageId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java index 4dae625a7..79e57a923 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java @@ -66,6 +66,7 @@ public class FileStatusMapper { .lastIndexed(status.getLastIndexed()) .fileSize(status.getFileSize()) .fileErrorInfo(status.getFileErrorInfo()) + .componentMappingVersions(status.getComponentMappingVersions()) .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index e8a48209b..6b5e81fb3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; import org.apache.commons.lang3.StringUtils; import org.springframework.retry.support.RetryTemplate; +import org.springframework.scheduling.annotation.Async; import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -39,17 +40,12 @@ public class FileStatusProcessingUpdateService { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); switch (analyzeResult.getMessageType()) { - case SURROUNDING_TEXT_ANALYSIS: break; - case ANALYSE: - case REANALYSE: default: - retryTemplate.execute(retryContext -> { - log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount()); - fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult); - return null; - }); + + log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, 0); + fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult); if (!analyzeResult.isWasReanalyzed()) { indexingService.addToIndexingQueue(IndexMessageType.INSERT, dossier.getDossierTemplateId(), dossierId, fileId, 2); @@ -79,16 +75,16 @@ public class FileStatusProcessingUpdateService { retryTemplate.execute(retryContext -> { log.info("Preprocessing dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount()); fileStatusService.setStatusPreProcessing(fileId, - fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); + fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); return null; }); var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId); if (updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()) { throw new ConflictException(String.format("Max Processing Retries exhausted for dossier %s and file %s with errorCount: %s", - dossierId, - fileId, - updatedFileEntity.getProcessingErrorCounter())); + dossierId, + fileId, + updatedFileEntity.getProcessingErrorCounter())); } } @@ -106,10 +102,10 @@ public class FileStatusProcessingUpdateService { retryTemplate.execute(retryContext -> { log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", - retryContext.getRetryCount(), - fileId, - dossierId, - fileErrorInfo != null ? fileErrorInfo.getCause() : null); + retryContext.getRetryCount(), + fileId, + dossierId, + fileErrorInfo != null ? fileErrorInfo.getCause() : null); fileStatusService.setStatusError(dossierId, fileId, fileErrorInfo); return null; }); @@ -123,7 +119,7 @@ public class FileStatusProcessingUpdateService { ocrFailed(dossierId, fileId, fileErrorInfo); } else { fileStatusService.setStatusOcrProcessing(fileId, - fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); + fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); fileStatusService.addToOcrQueue(dossierId, fileId, 2); } } @@ -136,7 +132,7 @@ public class FileStatusProcessingUpdateService { retryTemplate.execute(retryContext -> { log.info("Ocr processing dossier {} and file {}, Attempt to update status: {}", fileEntity.getDossierId(), fileId, retryContext.getRetryCount()); fileStatusService.setStatusOcrProcessing(fileId, - fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); + fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); return null; }); } @@ -149,9 +145,9 @@ public class FileStatusProcessingUpdateService { var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId); if (updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()) { throw new ConflictException(String.format("Max Ocr Retries exhausted for dossier %s and file %s with errorCount: %s", - updatedFileEntity.getDossierId(), - fileId, - updatedFileEntity.getProcessingErrorCounter())); + updatedFileEntity.getDossierId(), + fileId, + updatedFileEntity.getProcessingErrorCounter())); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 9818fefee..0440de55f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; + import org.apache.commons.lang3.StringUtils; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; @@ -90,6 +91,8 @@ public class FileStatusService { ViewedPagesPersistenceService viewedPagesPersistenceService; FileManagementServiceSettings fileManagementServiceSettings; LayoutParsingRequestFactory layoutParsingRequestFactory; + ComponentMappingService componentMappingService; + WebsocketService websocketService; @Transactional @@ -252,6 +255,7 @@ public class FileStatusService { .sectionsToReanalyse(sectionsToReanalyse) .fileId(fileId) .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted())) + .componentMappings(componentMappingService.getMetaDataByDossierTemplateId(dossierTemplate.getId())) .dossierTemplateId(dossier.getDossierTemplateId()) .lastProcessed(fileModel.getLastProcessed()) .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) @@ -517,7 +521,8 @@ public class FileStatusService { analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(), - analyzeResult.getAnalysisNumber()); + analyzeResult.getAnalysisNumber(), + analyzeResult.getUsedComponentMappings()); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java index 453f1e7ea..0857d3115 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java @@ -7,8 +7,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.service. import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; @@ -19,23 +22,28 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ReanalysisRequiredStatusService { - private final DictionaryPersistenceService dictionaryPersistenceService; - private final RulesPersistenceService rulesPersistenceService; - private final DossierPersistenceService dossierPersistenceService; - private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + DictionaryPersistenceService dictionaryPersistenceService; + RulesPersistenceService rulesPersistenceService; + DossierPersistenceService dossierPersistenceService; + LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + ComponentMappingService componentMappingService; public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) { @@ -78,8 +86,12 @@ public class ReanalysisRequiredStatusService { fileStatus.setDossierTemplateId(dossier.getDossierTemplateId()); fileStatus.setDossierStatusId(dossier.getDossierStatusId()); + if (fileStatus.isSoftOrHardDeleted()) { + log.debug("File {} is deleted, thus analysis is not required", fileStatus.getId()); + return new AnalysisRequiredResult(false, false); + } if (dossier.getSoftDeletedTime() != null || dossier.getHardDeletedTime() != null || dossier.getArchivedTime() != null) { - log.info("Dossier {} is deleted, thus analysis is not required", fileStatus.getDossierId()); + log.debug("Dossier {} is deleted, thus analysis is not required", fileStatus.getDossierId()); return new AnalysisRequiredResult(false, false); } @@ -118,39 +130,141 @@ public class ReanalysisRequiredStatusService { Map dossierVersionMap) { // get relevant versions - var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), k -> buildVersionData(dossier.getDossierTemplateId())); - var dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId())); + Map dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), + k -> buildVersionData(dossier.getDossierTemplateId())); + + Map componentMappingVersions = componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId()) + .stream() + .collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion)); + + Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId())); // compute matches + var mappingVersionAllMatch = mappingVersionsEqual(fileStatus, componentMappingVersions); var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L); var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L); var dictionaryVersionMatches = fileStatus.getDictionaryVersion() == dossierTemplateVersions.getOrDefault(DICTIONARY, -1L); var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L); var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion; - var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches; + var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch; var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches; + if (reanalysisRequired || fullAnalysisRequired) { - log.info( - "For file: {}-{} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}", - fileStatus.getId(), - fileStatus.getFilename(), - fileStatus.getRulesVersion(), - dossierTemplateVersions.getOrDefault(RULES, -1L), - fileStatus.getComponentRulesVersion(), - dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L), - fileStatus.getDictionaryVersion(), - dossierTemplateVersions.getOrDefault(DICTIONARY, -1L), - fileStatus.getLegalBasisVersion(), - dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L), - Math.max(fileStatus.getDossierDictionaryVersion(), 0), - dossierDictionaryVersion); + + log.info(buildMessage(fileStatus, + rulesVersionMatches, + dossierTemplateVersions, + componentRulesVersionMatches, + dictionaryVersionMatches, + legalBasisVersionMatches, + dossierDictionaryVersionMatches, + dossierDictionaryVersion, + mappingVersionAllMatch, + componentMappingVersions)); } return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired); } + private String buildMessage(FileModel fileStatus, + boolean rulesVersionMatches, + Map dossierTemplateVersions, + boolean componentRulesVersionMatches, + boolean dictionaryVersionMatches, + boolean legalBasisVersionMatches, + boolean dossierDictionaryVersionMatches, + Long dossierDictionaryVersion, + boolean mappingVersionAllMatch, + Map componentMappingVersions) { + + StringBuilder messageBuilder = new StringBuilder(); + messageBuilder.append("For file: ").append(fileStatus.getId()).append("-").append(fileStatus.getFilename()).append(" analysis is required because -> "); + + boolean needComma = false; + + if (!rulesVersionMatches) { + messageBuilder.append("ruleVersions: ").append(fileStatus.getRulesVersion()).append("/").append(dossierTemplateVersions.getOrDefault(RULES, -1L)); + needComma = true; + } + + if (!componentRulesVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("componentRuleVersions: ") + .append(fileStatus.getComponentRulesVersion()) + .append("/") + .append(dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L)); + needComma = true; + } + + if (!dictionaryVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("dictionaryVersions: ").append(fileStatus.getDictionaryVersion()).append("/").append(dossierTemplateVersions.getOrDefault(DICTIONARY, -1L)); + needComma = true; + } + + if (!legalBasisVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("legalBasisVersions: ").append(fileStatus.getLegalBasisVersion()).append("/").append(dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L)); + needComma = true; + } + + if (!dossierDictionaryVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("dossierDictionaryVersions: ").append(Math.max(fileStatus.getDossierDictionaryVersion(), 0)).append("/").append(dossierDictionaryVersion); + needComma = true; + } + + if (!mappingVersionAllMatch) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("componentMappingVersions: ").append(buildMappingVersionMatchesString(fileStatus.getComponentMappingVersions(), componentMappingVersions)); + needComma = true; + } + + return messageBuilder.toString(); + } + + + private String buildMappingVersionMatchesString(Map fileComponentMappingVersions, Map dbComponentMappingVersions) { + + Set allMappingNames = new HashSet<>(); + allMappingNames.addAll(fileComponentMappingVersions.keySet()); + allMappingNames.addAll(dbComponentMappingVersions.keySet()); + + StringBuilder sb = new StringBuilder(); + allMappingNames.stream() + .sorted() + .forEach(mappingName -> { + long fileVersion = fileComponentMappingVersions.getOrDefault(mappingName, -1); + long dbVersion = dbComponentMappingVersions.getOrDefault(mappingName, -1); + if (fileVersion != dbVersion) { + sb.append(mappingName).append(": ").append(fileVersion).append("/").append(dbVersion); + } + }); + return sb.toString(); + } + + + private static boolean mappingVersionsEqual(FileModel fileStatus, Map componentMappingVersions) { + + return fileStatus.getComponentMappingVersions().keySet().equals(componentMappingVersions.keySet()) && fileStatus.getComponentMappingVersions().keySet() + .stream() + .allMatch(name -> fileStatus.getComponentMappingVersions() + .get(name).equals(componentMappingVersions.get(name))); + } + + private Map buildVersionData(String dossierTemplateId) { var versions = new HashMap(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java index 166c86fa7..03d332582 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java @@ -4,16 +4,20 @@ import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicC import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; +import org.springframework.util.FileSystemUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -24,8 +28,10 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping; import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; @@ -43,6 +49,7 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.FileAttributesGeneralConfiguration; @@ -79,6 +86,7 @@ public class DossierTemplateExportService { private final RulesPersistenceService rulesPersistenceService; private final FileManagementStorageService fileManagementStorageService; private final ReportTemplatePersistenceService reportTemplatePersistenceService; + private final ComponentMappingService componentMappingService; private final ColorsService colorsService; private final EntryPersistenceService entryPersistenceService; private final ObjectMapper objectMapper = new ObjectMapper(); @@ -117,6 +125,7 @@ public class DossierTemplateExportService { objectMapper.registerModule(new JavaTimeModule()); DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId()); + downloadStatusPersistenceService.updateStatus(downloadJob.getStorageId(), DownloadStatusValue.GENERATING); String dossierTemplateId = extractDossierTemplateId(downloadStatus.getFilename()); var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId); @@ -234,6 +243,26 @@ public class DossierTemplateExportService { writeEntriesListToFile(fileSystemBackedArchiver, falseRecommendationValuesList, typeEntity.getType(), getFilename(ExportFilename.FALSE_RECOMMENDATION, TXT_EXT)); } + Path mappingDir = Files.createTempDirectory("mappings"); + + List componentMappings = componentMappingService.getMappingFilesByDossierTemplateId(dossierTemplateId, mappingDir); + + List metadata = componentMappings.stream() + .map(ComponentMapping::metaData) + .toList(); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename(), + ExportFilename.COMPONENT_MAPPINGS_FILE.getFilename() + JSON_EXT, + objectMapper.writeValueAsBytes(metadata))); + for (ComponentMapping componentMapping : componentMappings) { + try (var in = new FileInputStream(componentMapping.file())) { + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(ExportFilename.COMPONENT_MAPPINGS_FOLDER.getFilename(), + componentMapping.metaData().getName() + ".csv", + in.readAllBytes())); + } + } + + FileSystemUtils.deleteRecursively(mappingDir); + storeZipFile(downloadStatus.getStorageId(), fileSystemBackedArchiver); downloadStatusPersistenceService.updateStatus(downloadStatus, DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index ebc985b83..970b1a173 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.job; import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -69,7 +70,9 @@ public class AutomaticAnalysisJob implements Job { if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) { log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId()); return; - }var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); + } + + var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); if (redactionQueueInfo != null) { log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", tenant.getTenantId(), @@ -80,17 +83,17 @@ public class AutomaticAnalysisJob implements Job { var consumerCount = redactionQueueInfo.getConsumerCount(); if (redactionQueueInfo.getMessageCount() <= consumerCount * 5) { // queue up 5 files - var allStatuses = getAllRelevantStatuses(); + List allStatuses = getAllRelevantStatuses(); allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated)); - var allStatusesIterator = allStatuses.iterator(); + Iterator allStatusesIterator = allStatuses.iterator(); log.debug("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size()); - var queuedFiles = 0; + int queuedFiles = 0; while (queuedFiles < (consumerCount * 5) && allStatusesIterator.hasNext()) { - var next = allStatusesIterator.next(); + FileModel next = allStatusesIterator.next(); // in case the file doesn't have numberOfPages set, we assume an average. reanalyseFile(next); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index d47301caa..16b45a1c9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -2,9 +2,12 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -14,6 +17,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier; @@ -23,10 +27,12 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Websock import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; +import jakarta.persistence.EntityManager; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -42,6 +48,7 @@ public class FileStatusPersistenceService { private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; private final DossierPersistenceService dossierService; private final WebsocketService websocketService; + private final EntityManager entityManager; public void createStatus(String dossierId, String fileId, String filename, String uploader, long size) { @@ -61,6 +68,7 @@ public class FileStatusPersistenceService { file.setFileManipulationDate(now); file.setProcessingErrorCounter(0); file.setFileSize(size); + file.setComponentMappingVersions(new ArrayList<>()); fileRepository.save(file); } @@ -91,21 +99,15 @@ public class FileStatusPersistenceService { private int calculateProcessingErrorCounter(String fileId, ProcessingStatus processingStatus) { - switch (processingStatus) { - case ERROR: - return fileRepository.findById(fileId) - .map(FileEntity::getProcessingErrorCounter) - .orElse(0) + 1; - - case PROCESSED: - case REPROCESS: - return 0; - - default: - return fileRepository.findById(fileId) - .map(FileEntity::getProcessingErrorCounter) - .orElse(0); - } + return switch (processingStatus) { + case ERROR -> fileRepository.findById(fileId) + .map(FileEntity::getProcessingErrorCounter) + .orElse(0) + 1; + case PROCESSED, REPROCESS -> 0; + default -> fileRepository.findById(fileId) + .map(FileEntity::getProcessingErrorCounter) + .orElse(0); + }; } @@ -121,11 +123,13 @@ public class FileStatusPersistenceService { long duration, long dossierDictionaryVersion, int analysisVersion, - int analysisNumber) { + int analysisNumber, + List usedComponentMappings) { if (isFileDeleted(fileId)) { return; } + fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, @@ -141,10 +145,30 @@ public class FileStatusPersistenceService { analysisNumber, calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED)); + // must be modifiable, otherwise hibernate fails + List versionEntities = getFileEntityComponentMappingVersionEntities(usedComponentMappings); + + FileEntity fileEntity = entityManager.find(FileEntity.class, fileId); + + fileEntity.setComponentMappingVersions(versionEntities); + + entityManager.merge(fileEntity); + websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.FINISHED, analysisNumber); } + private static List getFileEntityComponentMappingVersionEntities(List usedComponentMappings) { + + if (usedComponentMappings == null) { + return Collections.emptyList(); + } + return usedComponentMappings.stream() + .map(cm -> new FileEntityComponentMappingVersionEntity(cm.getName(), cm.getVersion())) + .collect(Collectors.toList()); + } + + @Transactional public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ComponentMappingRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ComponentMappingRepository.java new file mode 100644 index 000000000..219d6866d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ComponentMappingRepository.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; + +import java.util.List; +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; + +import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DigitalSignatureEntity; + +public interface ComponentMappingRepository extends JpaRepository { + + List findByDossierTemplateId(String dossierTemplateId); + + + boolean existsByNameAndDossierTemplateId(String name, String dossierTemplateId); + + + Optional findByIdAndDossierTemplateId(String id, String dossierTemplateId); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 775564d99..6379bbce2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -47,12 +47,21 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " - + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, " - + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " - + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " - + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " - + "where f.id = :fileId") + @Query(""" + update FileEntity f set f.numberOfPages = :numberOfPages, \ + f.processingStatus = :processingStatus, \ + f.dictionaryVersion = :dictionaryVersion, \ + f.rulesVersion = :rulesVersion, \ + f.componentRulesVersion = :componentRulesVersion, \ + f.legalBasisVersion = :legalBasisVersion, \ + f.analysisDuration = :analysisDuration, \ + f.dossierDictionaryVersion = :dossierDictionaryVersion, \ + f.analysisVersion = :analysisVersion, \ + f.numberOfAnalyses = :analysisNumber, \ + f.lastUpdated = :lastUpdated, \ + f.lastProcessed = :lastProcessed, \ + f.processingErrorCounter = :processingErrorCounter \ + where f.id = :fileId""") void updateProcessingStatus(@Param("fileId") String fileId, @Param("numberOfPages") int numberOfPages, @Param("processingStatus") ProcessingStatus processingStatus, @@ -168,12 +177,13 @@ public interface FileRepository extends JpaRepository { @Param("lastUpdated") OffsetDateTime lastUpdated, @Param("softDeletedTime") OffsetDateTime softDeletedTime); + @Modifying @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id in (:fileIds)") int softDeleteFiles(@Param("fileIds") List fileIds, - @Param("processingStatus") ProcessingStatus processingStatus, - @Param("lastUpdated") OffsetDateTime lastUpdated, - @Param("softDeletedTime") OffsetDateTime softDeletedTime); + @Param("processingStatus") ProcessingStatus processingStatus, + @Param("lastUpdated") OffsetDateTime lastUpdated, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); @Modifying @@ -380,10 +390,7 @@ public interface FileRepository extends JpaRepository { + " when f.deleted is not null then f.deleted " + "end " + "where f.id in (:fileIds)") - int hardDeleteFiles(@Param("fileIds") List fileIds, - @Param("processingStatus") ProcessingStatus processingStatus, - @Param("deletionTime") OffsetDateTime deletionTime); - + int hardDeleteFiles(@Param("fileIds") List fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("deletionTime") OffsetDateTime deletionTime); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java index 7730046c4..ab16ad7f1 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java @@ -53,7 +53,7 @@ public class OCRProcessingMessageReceiver { response.getNumberOfOCRedPages()); } - log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE); + log.debug("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE); } @@ -63,7 +63,7 @@ public class OCRProcessingMessageReceiver { var response = objectMapper.readValue(failedMessage.getBody(), OCRStatusUpdateResponse.class); - log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL); + log.debug("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java index 8f6fbc97a..c1d89f926 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java @@ -1,7 +1,12 @@ package com.iqser.red.service.persistence.management.v1.processor.utils; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; @@ -14,6 +19,18 @@ public class FileModelMapper implements BiConsumer { fileEntity.getFileAttributes() .forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue())); fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp())); + fileModel.setComponentMappingVersions(getComponentMappingVersions(fileEntity)); + } + + + private static Map getComponentMappingVersions(FileEntity fileEntity) { + + if (Objects.isNull(fileEntity.getComponentMappingVersions())) { + return Collections.emptyMap(); + } + return fileEntity.getComponentMappingVersions() + .stream() + .collect(Collectors.toMap(FileEntityComponentMappingVersionEntity::getName, FileEntityComponentMappingVersionEntity::getVersion)); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index f01c902e0..470f13643 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -202,4 +202,8 @@ databaseChangeLog: - include: file: db/changelog/tenant/125-add-max-size-for-legal-basis-in-manual-legal-basis-change.yaml - include: - file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml \ No newline at end of file + file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml + - include: + file: db/changelog/tenant/127-add-component-mapping-table.yaml + - include: + file: db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/127-add-component-mapping-table.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/127-add-component-mapping-table.yaml new file mode 100644 index 000000000..e9679364e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/127-add-component-mapping-table.yaml @@ -0,0 +1,97 @@ +databaseChangeLog: + - changeSet: + id: create-table-component_mappings + author: kilian + changes: + - createTable: + tableName: component_mappings + columns: + - column: + name: id + type: VARCHAR(255) + constraints: + primaryKey: true + - column: + name: dossier_template_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: storage_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: name + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: file_name + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: version + type: integer + constraints: + nullable: false + defaultValue: '-1' + - column: + name: changed_date + type: TIMESTAMP WITH TIME ZONE + constraints: + nullable: false + - column: + name: number_of_lines + type: INTEGER + constraints: + nullable: false + defaultValue: '0' + - column: + name: encoding + type: VARCHAR(255) + constraints: + nullable: false + defaultValue: 'UTF-8' + - column: + name: delimiter + type: CHAR(1) + constraints: + nullable: false + defaultValue: ',' + - addUniqueConstraint: + columnNames: name, dossier_template_id + constraintName: unique_name_dossier_template + tableName: component_mappings + - addForeignKeyConstraint: + baseColumnNames: dossier_template_id + baseTableName: component_mappings + constraintName: fk_entity_dossier_template_id + referencedColumnNames: id + referencedTableName: dossier_template + onDelete: CASCADE + onUpdate: CASCADE + + - createTable: + tableName: component_mapping_column_labels + columns: + - column: + name: component_mapping_entity_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: column_labels + type: VARCHAR(255) + constraints: + nullable: false + + - addForeignKeyConstraint: + baseColumnNames: component_mapping_entity_id + baseTableName: component_mapping_column_labels + constraintName: fk_component_mapping_column_labels_entity_id + referencedColumnNames: id + referencedTableName: component_mappings + onDelete: CASCADE + onUpdate: CASCADE diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml new file mode 100644 index 000000000..e9dbcefa2 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml @@ -0,0 +1,31 @@ +databaseChangeLog: + - changeSet: + id: add_component_mapping_version_to_file_entity + author: kilian + changes: + - createTable: + tableName: file_entity_component_mapping_versions + columns: + - column: + name: file_entity_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: name + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: version + type: INT + constraints: + nullable: false + - addForeignKeyConstraint: + baseColumnNames: file_entity_id + baseTableName: file_entity_component_mapping_versions + constraintName: fk_component_mapping_version_entity_file + referencedColumnNames: id + referencedTableName: file + onDelete: CASCADE + onUpdate: CASCADE diff --git a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts index 19ed8474c..6e14c8152 100644 --- a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts @@ -27,6 +27,7 @@ dependencies { api("junit:junit:4.13.2") api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0") api("net.logstash.logback:logstash-logback-encoder:7.4") + testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2") testImplementation("org.springframework.security:spring-security-test:6.0.2") testImplementation("org.testcontainers:postgresql:1.17.1") diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index 92332f560..175e5ee92 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import org.apache.commons.codec.binary.Base64; @@ -115,6 +116,7 @@ public class FileTesterAndProvider { result.setDossierDictionaryVersion(1); result.setAnalysisNumber(1); result.setAnalysisVersion(1); + result.setUsedComponentMappings(Collections.emptyList()); fileStatusService.setStatusSuccessful(dossierId, fileId, result); fileStatusService.setStatusProcessed(fileId); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/CustomPermissionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/CustomPermissionTest.java index 38f5d7d97..50bacb882 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/CustomPermissionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/CustomPermissionTest.java @@ -10,10 +10,6 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import com.iqser.red.service.peristence.v1.server.integration.client.CustomPermissionClient; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java index cc9497ce1..19d4f8ced 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java @@ -26,6 +26,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService; @@ -73,6 +74,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { @MockBean private ColorsService colorsService; @MockBean + private ComponentMappingService componentMappingService; + @MockBean private StorageService storageService; @MockBean private DossierStatusPersistenceService dossierStatusPersistenceService; @@ -125,7 +128,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { storageService, dossierStatusPersistenceService, watermarkService, - fileManagementStorageService); + fileManagementStorageService, + componentMappingService); dossierTemplateExportService = new DossierTemplateExportService(dossierTemplatePersistenceService, downloadStatusPersistenceService, dossierAttributeConfigPersistenceService, @@ -136,6 +140,7 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { rulesPersistenceService, fileManagementStorageService, reportTemplatePersistenceService, + componentMappingService, colorsService, entryPersistenceService, watermarkService, diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneServiceTest.java index fd9846deb..d783fd0fd 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneServiceTest.java @@ -28,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService; @@ -91,6 +92,8 @@ public class DossierTemplateCloneServiceTest { private FileManagementStorageService fileManagementStorageService; @MockBean private TypeRepository typeRepository; + @MockBean + private ComponentMappingService componentMappingService; private DossierTemplateCloneService dossierTemplateCloneService; @@ -124,7 +127,8 @@ public class DossierTemplateCloneServiceTest { storageService, dossierStatusPersistenceService, watermarkService, - fileManagementStorageService); + fileManagementStorageService, + componentMappingService); dummyTemplate = new DossierTemplateEntity(); setNonDefaultValues(dummyTemplate); @@ -169,7 +173,7 @@ public class DossierTemplateCloneServiceTest { if (getterMethod != null) { value = getterMethod.invoke(obj); - } else if (alternativeGetterMethod != null){ + } else if (alternativeGetterMethod != null) { value = alternativeGetterMethod.invoke(obj); } else { return; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java new file mode 100644 index 000000000..4ca22ffc9 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportExportTest.java @@ -0,0 +1,173 @@ +package com.iqser.red.service.peristence.v1.server.integration.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.util.FileSystemUtils; + +import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; +import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService; +import com.iqser.red.service.persistence.management.v1.processor.service.export.DossierTemplateExportService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.SneakyThrows; + +public class DossierTemplateImportExportTest extends AbstractPersistenceServerServiceTest { + + public static final String IMPORTED_TEMPLATE_NAME = "imported-template"; + public static final String USER_ID = "NutzerIdentifikationsNummer"; + public static final String AFTER = "after"; + public static final String BEFORE = "before"; + + @Autowired + DossierTemplateImportService dossierTemplateImportService; + + @Autowired + DossierTemplateExportService dossierTemplateExportService; + + @Autowired + StorageService storageService; + + @MockBean + DownloadStatusPersistenceService downloadStatusPersistenceService; + + + @AfterEach + public void clearStorage() { + + ((FileSystemBackedStorageService) storageService).clearStorage(); + } + + + @Test + @SneakyThrows +// @DirtiesContext + public void testImportExportRoundtrip() { + + TenantContext.setTenantId(TENANT_1); + Path outDir = Files.createTempDirectory(IMPORTED_TEMPLATE_NAME); + + Path dossierTemplateExportArchive = new ClassPathResource("files/dossiertemplates/DossierTemplate.zip").getFile().toPath(); + + String importedDossierTemplateId = dossierTemplateImportService.importDossierTemplate(ImportDossierTemplateRequest.builder() + .archive(Files.readAllBytes(dossierTemplateExportArchive)) + .userId(USER_ID) + .build()); + + when(downloadStatusPersistenceService.getStatus(anyString()))// + .thenReturn(DownloadStatusEntity.builder().filename(importedDossierTemplateId + ".zip").storageId(IMPORTED_TEMPLATE_NAME).build()); + + dossierTemplateExportService.createDownloadArchive(DownloadJob.builder().storageId(importedDossierTemplateId).userId(USER_ID).build()); + + File tmpFile = Files.createTempFile(IMPORTED_TEMPLATE_NAME, ".zip").toFile(); + storageService.downloadTo(TenantContext.getTenantId(), IMPORTED_TEMPLATE_NAME, tmpFile); + + unzip(dossierTemplateExportArchive.toFile().toString(), outDir.resolve(BEFORE)); + unzip(tmpFile.toString(), outDir.resolve(AFTER)); + + assert tmpFile.delete(); + + Map beforeContents = getDirectoryContents(outDir.resolve(BEFORE)); + Map afterContents = getDirectoryContents(outDir.resolve(AFTER)); + + assertEquals(beforeContents.size(), afterContents.size()); + assertEquals(beforeContents.keySet(), afterContents.keySet()); + + // can't assert equality on contents as UUID's are supposed to change on import + + FileSystemUtils.deleteRecursively(outDir); + } + + + public static void unzip(String zipFilePath, Path destDirectory) throws IOException { + + File destDir = destDirectory.toFile(); + if (!destDir.exists()) { + destDir.mkdir(); + } + try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath))) { + ZipEntry entry = zipIn.getNextEntry(); + while (entry != null) { + String filePath = destDirectory + File.separator + entry.getName(); + if (!entry.isDirectory()) { + // if the entry is a file, extracts it + extractFile(zipIn, filePath); + } else { + // if the entry is a directory, make the directory + File dir = new File(filePath); + dir.mkdirs(); + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); + } + } + } + + + private static void extractFile(ZipInputStream zipIn, String filePath) throws IOException { + + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) { + byte[] bytesIn = new byte[4096]; + int read; + while ((read = zipIn.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + } + } + + + private static Map getDirectoryContents(Path dir) throws IOException, NoSuchAlgorithmException { + + Map contents = new HashMap<>(); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + + Files.walkFileTree(dir, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + + contents.put(dir.relativize(file), digest.digest(Files.readAllBytes(file))); + return FileVisitResult.CONTINUE; + } + + + @Override + public FileVisitResult preVisitDirectory(Path subDir, BasicFileAttributes attrs) { + + contents.put(dir.relativize(subDir), new byte[0]); + return FileVisitResult.CONTINUE; + } + }); + + return contents; + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java index 55535290d..110b05765 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -298,7 +299,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k); if (k == 1) { - fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1); + fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1, Collections.emptyList()); reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k)))); } if (k == 2) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index 8f9d9a87b..ce27b8a53 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -9,14 +9,17 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository; + import org.assertj.core.util.Lists; import org.bson.BsonArray; import org.bson.BsonDocument; import org.bson.BsonString; +import org.checkerframework.checker.units.qual.A; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; @@ -38,6 +41,7 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; +import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.StatementCallback; import org.springframework.jdbc.datasource.SingleConnectionDataSource; @@ -50,8 +54,10 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.provisioning.InMemoryUserDetailsManager; import org.springframework.security.web.SecurityFilterChain; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.support.TestPropertySourceUtils; import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.peristence.v1.server.Application; @@ -138,6 +144,9 @@ import lombok.extern.slf4j.Slf4j; @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = "spring-hibernate-query-utils.n-plus-one-queries-detection.error-level=INFO") public abstract class AbstractPersistenceServerServiceTest { + public static final String TENANT_1 = "redaction"; + public static final String TENANT_2 = "redaction2"; + @MockBean protected Scheduler scheduler; @MockBean @@ -273,7 +282,7 @@ public abstract class AbstractPersistenceServerServiceTest { @BeforeEach public void setupTenantContext() { - TenantContext.setTenantId("redaction"); + TenantContext.setTenantId(TENANT_1); } @@ -289,7 +298,7 @@ public abstract class AbstractPersistenceServerServiceTest { createTenants(); - TenantContext.setTenantId("redaction"); + TenantContext.setTenantId(TENANT_1); ApplicationConfig appConfig = ApplicationConfig.builder().downloadCleanupDownloadFilesHours(8).downloadCleanupNotDownloadFilesHours(72).softDeleteCleanupTime(96).build(); applicationConfigService.saveApplicationConfiguration(MagicConverter.convert(appConfig, ApplicationConfigurationEntity.class)); @@ -346,9 +355,9 @@ public abstract class AbstractPersistenceServerServiceTest { if (tenantsClient.getTenants() == null || tenantsClient.getTenants().isEmpty()) { var redactionTenant = new TenantResponse(); - redactionTenant.setTenantId("redaction"); - redactionTenant.setGuid("redaction"); - redactionTenant.setDisplayName("redaction"); + redactionTenant.setTenantId(TENANT_1); + redactionTenant.setGuid(TENANT_1); + redactionTenant.setDisplayName(TENANT_1); redactionTenant.setAuthDetails(new AuthDetails()); redactionTenant.setDatabaseConnection(DatabaseConnection.builder() .driver("postgresql") @@ -388,9 +397,9 @@ public abstract class AbstractPersistenceServerServiceTest { .build()); var redactionTenant2 = new TenantResponse(); - redactionTenant2.setTenantId("redaction2"); - redactionTenant2.setGuid("redaction2"); - redactionTenant2.setDisplayName("redaction2"); + redactionTenant2.setTenantId(TENANT_2); + redactionTenant2.setGuid(TENANT_2); + redactionTenant2.setDisplayName(TENANT_2); redactionTenant2.setAuthDetails(new AuthDetails()); redactionTenant2.setDatabaseConnection(DatabaseConnection.builder() .driver("postgresql") @@ -425,19 +434,19 @@ public abstract class AbstractPersistenceServerServiceTest { .username(MONGO_USERNAME) .password(encryptionDecryptionService.encrypt(MONGO_PASSWORD)) .address(mongoDbContainer.getHost() + ":" + mongoDbContainer.getFirstMappedPort()) - .database("redaction2") + .database(TENANT_2) .options("") .build()); - when(tenantsClient.getTenant("redaction")).thenReturn(redactionTenant); - when(tenantsClient.getTenant("redaction2")).thenReturn(redactionTenant2); + when(tenantsClient.getTenant(TENANT_1)).thenReturn(redactionTenant); + when(tenantsClient.getTenant(TENANT_2)).thenReturn(redactionTenant2); when(tenantsClient.getTenants()).thenReturn(List.of(redactionTenant, redactionTenant2)); try { - tenantCreatedListener.createTenant(new TenantCreatedEvent("redaction")); - tenantCreatedListener.createTenant(new TenantCreatedEvent("redaction2")); - mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent("redaction")); - mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent("redaction2")); + tenantCreatedListener.createTenant(new TenantCreatedEvent(TENANT_1)); + tenantCreatedListener.createTenant(new TenantCreatedEvent(TENANT_2)); + mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent(TENANT_1)); + mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent(TENANT_2)); } catch (Exception e) { e.printStackTrace(); @@ -515,7 +524,6 @@ public abstract class AbstractPersistenceServerServiceTest { notificationPreferencesRepository.deleteAll(); indexInformationRepository.deleteAll(); applicationConfigRepository.deleteAll(); - entityLogEntryDocumentRepository.deleteAll(); entityLogDocumentRepository.deleteAll(); @@ -526,8 +534,16 @@ public abstract class AbstractPersistenceServerServiceTest { @Slf4j static class Initializer implements ApplicationContextInitializer { + static AtomicInteger UniquePortFactory = new AtomicInteger(28081); + + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + // any time the WebEnvironment is re-instantiated use a new port, as it sometimes leads to PortBindExceptions + ConfigurableEnvironment environment = configurableApplicationContext.getEnvironment(); + int port = UniquePortFactory.getAndIncrement(); + TestPropertySourceUtils.addInlinedPropertiesToEnvironment(environment, "server.port=" + port); + var postgreSQLContainerMaster = SpringPostgreSQLTestContainer.getInstance().withDatabaseName("integration-tests-db-master").withUsername("sa").withPassword("sa"); postgreSQLContainerMaster.start(); @@ -537,8 +553,8 @@ public abstract class AbstractPersistenceServerServiceTest { var mongoInstance = MongoDBTestContainer.getInstance(); mongoInstance.start(); - createMongoDBDatabase(mongoInstance, "redaction"); - createMongoDBDatabase(mongoInstance, "redaction2"); + createMongoDBDatabase(mongoInstance, TENANT_1); + createMongoDBDatabase(mongoInstance, TENANT_2); log.info("Hosts are - Redis: {}, Postgres: {}, MongoDB: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost(), mongoInstance.getHost()); @@ -553,37 +569,36 @@ public abstract class AbstractPersistenceServerServiceTest { } - } + private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer, String databaseName) { - private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer, String databaseName) { + try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/", + MONGO_USERNAME, + MONGO_PASSWORD, + mongoDBTestContainer.getHost(), + mongoDBTestContainer.getFirstMappedPort()))) { + MongoDatabase database = mongoClient.getDatabase(databaseName); + BsonDocument createUserCommand = new BsonDocument(); + createUserCommand.append("createUser", new BsonString(MONGO_USERNAME)); + createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD)); + BsonArray roles = new BsonArray(); + roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(databaseName))); + createUserCommand.append("roles", roles); - try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/", - MONGO_USERNAME, - MONGO_PASSWORD, - mongoDBTestContainer.getHost(), - mongoDBTestContainer.getFirstMappedPort()))) { - MongoDatabase database = mongoClient.getDatabase(databaseName); - BsonDocument createUserCommand = new BsonDocument(); - createUserCommand.append("createUser", new BsonString(MONGO_USERNAME)); - createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD)); - BsonArray roles = new BsonArray(); - roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(databaseName))); - createUserCommand.append("roles", roles); + try { + database.runCommand(createUserCommand); + } catch (MongoCommandException mongoCommandException) { + // ignore user already exists + if (mongoCommandException.getErrorCode() != 51003) { + throw mongoCommandException; + } - try { - database.runCommand(createUserCommand); - } catch (MongoCommandException mongoCommandException) { - // ignore user already exists - if (mongoCommandException.getErrorCode() != 51003) { - throw mongoCommandException; } } - } - } + } @Configuration @EnableWebSecurity diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip new file mode 100644 index 000000000..bc076a829 Binary files /dev/null and b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/DossierTemplate.zip differ diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java index b00739042..29014cc47 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Set; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import lombok.AllArgsConstructor; import lombok.Builder; @@ -34,5 +35,8 @@ public class AnalyzeRequest { @Builder.Default private List fileAttributes = new ArrayList<>(); + @Builder.Default + private List componentMappings = new ArrayList<>(); + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java index 5ef8e7b29..f6148ec5b 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java @@ -1,8 +1,10 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; +import java.util.List; import java.util.Set; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import lombok.AllArgsConstructor; import lombok.Builder; @@ -37,4 +39,6 @@ public class AnalyzeResult { private Set addedFileAttributes; + private List usedComponentMappings; + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java index 1af0c17f7..d3baf9030 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java @@ -1,6 +1,7 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; import java.time.OffsetDateTime; +import java.util.Map; import java.util.Set; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; @@ -147,7 +148,8 @@ public class FileStatus { private OffsetDateTime lastIndexed; @Schema(description = "The error information for the error state of the file") private FileErrorInfo fileErrorInfo; - + @Schema(description = "Shows which versions of each mapping the last analysis has been performed") + private Map componentMappingVersions; @Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.") public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java new file mode 100644 index 000000000..410a74afd --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java @@ -0,0 +1,38 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.component; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ComponentMappingMetadata { + + String id; + + String name; + + String fileName; + + String storageId; + + Integer version; + + List columnLabels; + + Integer numberOfLines; + + String encoding; + + char delimiter; + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java index 06e45cee0..6794ba1aa 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java @@ -73,6 +73,7 @@ public class FileModel { private OffsetDateTime fileManipulationDate; private boolean hasHighlights; private FileErrorInfo fileErrorInfo; + private Map componentMappingVersions = new HashMap<>(); public long getFileSize() { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java index a2125b0c2..80112f08f 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java @@ -19,6 +19,5 @@ public enum ProcessingStatus { PRE_PROCESSED, FIGURE_DETECTION_ANALYZING, TABLE_PARSING_ANALYZING, - VISUAL_LAYOUT_PARSING_ANALYZING } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java new file mode 100644 index 000000000..1aa95b411 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ComponentMappingImportModel.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport; + +import java.io.File; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; + +public record ComponentMappingImportModel(ComponentMappingMetadata metadata, byte[] csvData) { + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java index aff455da0..905b4fed0 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ExportFilename.java @@ -19,7 +19,9 @@ public enum ExportFilename { DOSSIER_TYPE("dossierType"), ENTRIES("entries"), FALSE_POSITIVES("falsePositives"), - FALSE_RECOMMENDATION("falseRecommendations"); + FALSE_RECOMMENDATION("falseRecommendations"), + COMPONENT_MAPPINGS_FOLDER("mappings"), + COMPONENT_MAPPINGS_FILE("componentMappingsList"); @Getter private final String filename; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java index fe4a7e29a..6f20e791e 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/importexport/ImportTemplateResult.java @@ -50,6 +50,9 @@ public class ImportTemplateResult { @Builder.Default public List fileAttributesConfigs = new ArrayList<>(); + @Builder.Default + public List componentMappings = new ArrayList<>(); + @Builder.Default public List legalBases = new ArrayList<>(); diff --git a/publish-custom-image.sh b/publish-custom-image.sh index b680eb16a..5a0def0ea 100755 --- a/publish-custom-image.sh +++ b/publish-custom-image.sh @@ -1,5 +1,9 @@ #!/bin/bash + +set -e + dir=${PWD##*/} + gradle assemble # Get the current Git branch @@ -11,5 +15,31 @@ commit_hash=$(git rev-parse --short=5 HEAD) # Combine branch and commit hash buildName="${USER}-${branch}-${commit_hash}" -gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName -echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName" +gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${buildName} + +newImageName="nexus.knecon.com:5001/red/${dir}-server-v1:${buildName}" + +echo "full image name:" +echo ${newImageName} +echo "" + +if [ -z "$1" ]; then + exit 0 +fi + +namespace=${1} +deployment_name="persistence-service-v1" + +echo "deploying to ${namespace}" + +oldImageName=$(rancher kubectl -n ${namespace} get deployment ${deployment_name} -o=jsonpath='{.spec.template.spec.containers[*].image}') + +if [ "${newImageName}" = "${oldImageName}" ]; then + echo "Image tag did not change, redeploying..." + rancher kubectl rollout restart deployment ${deployment_name} -n ${namespace} +else + echo "upgrading the image tag..." + rancher kubectl set image deployment/${deployment_name} ${deployment_name}=${newImageName} -n ${namespace} +fi +rancher kubectl rollout status deployment ${deployment_name} -n ${namespace} +echo "Built ${deployment_name}:${buildName} and deployed to ${namespace}"