Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'feature/RED-10260' into 'master'

Browse Source 
RED-10260: add quote char to component mapping

Closes RED-10260

See merge request redactmanager/persistence-service!793
This commit is contained in:
Kilian Schüttler 2024-10-24 11:00:59 +02:00
commit ccbf977e01
12 changed files with 139 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java vendored
View File

@ -269,7 +269,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
@Override
@SneakyThrows
@PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter) {
public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter, String quoteChar) {
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
@ -285,18 +285,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
throw new BadRequestException(format("The provided file name \"%s\" is not valid!", nameToUse));
}
if (Strings.isNullOrEmpty(delimiter)) {
throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
} else if (delimiter.length() != 1) {
throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
}
char cleanDelimiter = delimiter.charAt(0);
char cleanDelimiter = getDelimiter(delimiter);
char cleanQuoteChar = getQuoteChar(quoteChar);
Path mappingFile = saveToFile(file);
try {
ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, cleanDelimiter, encoding, mappingFile.toFile());
ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
nameToUse,
fileName,
cleanDelimiter,
encoding,
mappingFile.toFile(),
cleanQuoteChar);
return componentMappingMapper.toModel(metaData);
} finally {
@ -309,18 +311,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
@Override
@SneakyThrows
@PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String name, String encoding, String delimiter) {
public ComponentMappingMetadataModel updateMapping(String dossierTemplateId,
String componentMappingId,
MultipartFile file,
String name,
String encoding,
String delimiter,
String quoteChar) {
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
String nameToUse = validateFileName(file, name);
if (Strings.isNullOrEmpty(delimiter)) {
throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
} else if (delimiter.length() != 1) {
throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
}
char cleanDelimiter = delimiter.charAt(0);
char cleanDelimiter = getDelimiter(delimiter);
char cleanQuoteChar = getQuoteChar(quoteChar);
Path mappingFile = saveToFile(file);
@ -331,7 +335,8 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
encoding,
cleanDelimiter,
mappingFile.toFile(),
file.getOriginalFilename());
file.getOriginalFilename(),
cleanQuoteChar);
return componentMappingMapper.toModel(resultMetaData);
} finally {
@ -340,6 +345,28 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
}
private static char getDelimiter(String delimiter) {
if (Strings.isNullOrEmpty(delimiter)) {
throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
} else if (delimiter.length() != 1) {
throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
}
return delimiter.charAt(0);
}
private static char getQuoteChar(String quoteChar) {
if (Strings.isNullOrEmpty(quoteChar)) {
throw new BadRequestException("The provided quoteChar is not valid! Can't be null or empty.");
} else if (quoteChar.length() != 1) {
throw new BadRequestException(format("The provided quoteChar %s is not valid! Only a single character is allowed.", quoteChar));
}
return quoteChar.charAt(0);
}
private static String validateFileName(MultipartFile file, String name) {
if (Strings.isNullOrEmpty(file.getOriginalFilename()) || !file.getOriginalFilename().endsWith(".csv")) {

10
persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java vendored
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.service.v2.api.external.resource;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DateFormatPatternErrorMessage;
import com.iqser.red.service.persistence.service.v2.api.external.model.DossierAttributeDefinitionList;
import com.iqser.red.service.persistence.service.v2.api.external.model.DossierStatusDefinitionList;
import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList;
@ -63,6 +62,7 @@ public interface DossierTemplateResource {
String DRY_RUN_PARAM = "dryRun";
String ENCODING_PARAM = "encoding";
String DELIMITER_PARAM = "delimiter";
String QUOTE_CHAR_PARAM = "quoteChar";
String MAPPING_NAME_PARAM = "name";
String INCLUDE_SOFT_DELETED = "includeSoftDeleted";
@ -127,7 +127,7 @@ public interface DossierTemplateResource {
@Operation(summary = "Upload a date formats file for a specific DossierTemplate.")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Date formats upload successful."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found."), @ApiResponse(responseCode = "400", description = "Uploaded date formats could not be verified."), @ApiResponse(responseCode = "422", description = "Uploaded date formats file could not be parsed.")})
ResponseEntity<?> uploadDateFormats(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
@ResponseBody
@ -159,7 +159,8 @@ public interface DossierTemplateResource {
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
@Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
@Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
@Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
@Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
@Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
@Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None")
@ -173,7 +174,8 @@ public interface DossierTemplateResource {
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
@Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
@Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
@Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
@Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
@Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
@ResponseBody

17
persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml
View File

@ -509,7 +509,7 @@ paths:
- **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
#### Customization Options
- Users can specify the delimiter and encoding used in the CSV file.
- Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
#### Usage
- The component mapping file can be utilized in component rules to relate components to existing master data.
@ -533,6 +533,7 @@ paths:
- $ref: '#/components/parameters/mappingName'
- $ref: '#/components/parameters/encoding'
- $ref: '#/components/parameters/delimiter'
- $ref: '#/components/parameters/quoteChar'
responses:
"200":
content:
@ -609,7 +610,7 @@ paths:
- **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
#### Customization Options
- Users can specify the delimiter and encoding used in the CSV file.
- Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
tags:
- 1. Dossier Templates
requestBody:
@ -623,6 +624,7 @@ paths:
- $ref: '#/components/parameters/mappingName'
- $ref: '#/components/parameters/encoding'
- $ref: '#/components/parameters/delimiter'
- $ref: '#/components/parameters/quoteChar'
responses:
"200":
content:
@ -2114,6 +2116,17 @@ components:
example: ','
default: ','
description: "The delimiter used as a separator in a csv file."
quoteChar:
name: quoteChar
required: false
in: query
schema:
type: string
minLength: 1
maxLength: 1
example: '"'
default: '"'
description: "The quoteChar used to quote fields in a csv file."
mappingName:
name: name
required: false

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateImportService.java
View File

@ -75,10 +75,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ -449,7 +447,8 @@ public class DossierTemplateImportService {
componentMapping.metadata().getFileName(),
componentMapping.metadata().getDelimiter(),
componentMapping.metadata().getEncoding(),
tmpFile);
tmpFile,
componentMapping.metadata().getQuoteChar());
componentMappingService.setVersion(createdMapping.getId(), componentMapping.metadata().getVersion());
assert tmpFile.delete();
@ -524,6 +523,7 @@ public class DossierTemplateImportService {
}
private LayoutParsingType deferFromApplicationType() {
return Objects.equals(applicationType, "DocuMine") ? LayoutParsingType.DOCUMINE_OLD : LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH;

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java
View File

@ -75,4 +75,7 @@ public class ComponentMappingEntity {
@Builder.Default
char delimiter = ',';
@Builder.Default
char quoteChar = '"';
}

39
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java
View File

@ -62,16 +62,23 @@ public class ComponentMappingService {
@SneakyThrows
public ComponentMappingMetadata update(String dossierTemplateId, String mappingId, String name, String encoding, char delimiter, File mappingFile, String fileName) {
public ComponentMappingMetadata update(String dossierTemplateId,
String mappingId,
String name,
String encoding,
char delimiter,
File mappingFile,
String fileName,
char quoteChar) {
ComponentMappingEntity entity = componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId);
return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName);
return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName, quoteChar);
}
@SneakyThrows
public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile) {
public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile, char quoteChar) {
if (componentMappingPersistenceService.existsByNameAndDossierTemplateId(name, dossierTemplateId)) {
throw new BadRequestException("A mapping with this name already exists in the dossier template!");
@ -86,20 +93,27 @@ public class ComponentMappingService {
.fileName(fileName)
.build();
return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName);
return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName, quoteChar);
}
@SneakyThrows
private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity, String name, String encoding, char delimiter, File mappingFile, String fileName) {
private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity,
String name,
String encoding,
char delimiter,
File mappingFile,
String fileName,
char quoteChar) {
Charset charset = resolveCharset(encoding);
CsvStats stats = sortCSVFile(delimiter, mappingFile, charset);
CsvStats stats = sortCSVFile(delimiter, mappingFile, charset, quoteChar);
entity.setName(name);
entity.setDelimiter(delimiter);
entity.setQuoteChar(quoteChar);
entity.setEncoding(encoding);
entity.setNumberOfLines(stats.numberOfLines());
entity.setColumnLabels(stats.columnLabels());
@ -126,7 +140,7 @@ public class ComponentMappingService {
}
private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws BadRequestException, IOException {
private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset, char quoteChar) throws BadRequestException, IOException {
Path tempFile = Files.createTempFile("mapping", ".tmp");
@ -135,11 +149,8 @@ public class ComponentMappingService {
String[] columnLabels;
int numberOfLines = 0;
try (Reader fileReader = new FileReader(tempFile.toFile(), charset);//
CSVReader reader = buildReader(fileReader, delimiter);//
CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset), delimiter,
CSVWriter.NO_QUOTE_CHARACTER,
CSVWriter.DEFAULT_ESCAPE_CHARACTER,
CSVWriter.DEFAULT_LINE_END)) {
CSVReader reader = buildReader(fileReader, delimiter, quoteChar);//
CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset), delimiter, quoteChar, '\\', CSVWriter.DEFAULT_LINE_END)) {
List<String[]> rows = reader.readAll();
@ -180,9 +191,9 @@ public class ComponentMappingService {
}
private static CSVReader buildReader(Reader reader, char delimiter) throws IOException {
private static CSVReader buildReader(Reader reader, char delimiter, char quoteChar) throws IOException {
return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build();
return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).withQuoteChar(quoteChar).build()).build();
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java
View File

@ -216,7 +216,8 @@ public class DossierTemplateCloneService {
componentMapping.metaData().getFileName(),
componentMapping.metaData().getDelimiter(),
componentMapping.metaData().getEncoding(),
componentMapping.file());
componentMapping.file(),
componentMapping.metaData().getQuoteChar());
}
FileSystemUtils.deleteRecursively(dir);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -235,3 +235,5 @@ databaseChangeLog:
file: db/changelog/tenant/145-add-indexes-to-file-table.yaml
- include:
file: db/changelog/tenant/146-add-layout-parsing-type-to-dossier-template.yaml
- include:
file: db/changelog/tenant/147-add-quotechar-to-component-mapping.yaml

14
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/147-add-quotechar-to-component-mapping.yaml Normal file
View File

@ -0,0 +1,14 @@
databaseChangeLog:
- changeSet:
id: add-quote_char-to-component-mapping
author: kilian
changes:
- addColumn:
tableName: component_mappings
columns:
- column:
name: quote_char
type: char
defaultValue: '"'
constraints:
nullable: false

24
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentMappingTest.java
View File

@ -53,7 +53,12 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
"application/csv",
IOUtils.toByteArray(new ClassPathResource("files/componentmapping/file.csv").getInputStream()));
ComponentMappingMetadataModel componentMappingMetadataModel = dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(), mockMultipartFile, "file", "UTF-8", ",");
ComponentMappingMetadataModel componentMappingMetadataModel = dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(),
mockMultipartFile,
"file",
"UTF-8",
",",
"\"");
assertEquals(componentMappingMetadataModel.getFileName(), mockMultipartFile.getOriginalFilename());
@ -65,11 +70,20 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
when(componentMappingPersistenceService.getEntityById(anyString(), anyString())).thenReturn(ComponentMappingEntity.builder()
.id(componentMappingMetadataModel.getId())
.dossierTemplate(new DossierTemplateEntity())
.storageId(buildStorageId(dossierTemplate.getId(), componentMappingMetadataModel.getId(), "file", "update_file.csv"))
.storageId(buildStorageId(dossierTemplate.getId(),
componentMappingMetadataModel.getId(),
"file",
"update_file.csv"))
.fileName("update_file.csv")
.build());
componentMappingMetadataModel = dossierTemplateExternalClient.updateMapping(dossierTemplate.getId(), componentMappingMetadataModel.getId(), updateMockMultipartFile, "file", "UTF-8", ",");
componentMappingMetadataModel = dossierTemplateExternalClient.updateMapping(dossierTemplate.getId(),
componentMappingMetadataModel.getId(),
updateMockMultipartFile,
"file",
"UTF-8",
",",
"\"");
assertEquals(componentMappingMetadataModel.getFileName(), updateMockMultipartFile.getOriginalFilename());
}
@ -86,7 +100,8 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
"application/csv",
IOUtils.toByteArray(new ClassPathResource("files/componentmapping/empty.csv").getInputStream()));
var result = assertThrows(FeignException.class, () -> dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(), mockMultipartFile, "file", "UTF-8", ","));
String id = dossierTemplate.getId();
var result = assertThrows(FeignException.class, () -> dossierTemplateExternalClient.uploadMapping(id, mockMultipartFile, "file", "UTF-8", ",", "\""));
assertTrue(result.getMessage().contains("CSV file can not be empty!"));
}
@ -95,4 +110,5 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
return dossierTemplateId + "/" + id + "_" + name + "_" + fileName;
}
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java
View File

@ -35,4 +35,6 @@ public class ComponentMappingMetadata {
char delimiter;
char quoteChar;
}

5
publish-custom-image.sh
View File

@ -9,11 +9,14 @@ gradle assemble
# Get the current Git branch
branch=$(git rev-parse --abbrev-ref HEAD)
# Replace any slashes (e.g., in 'feature/' or 'release/') with a hyphen
cleaned_branch=$(echo "$branch" | sed 's/\//_/g')
# Get the short commit hash (first 5 characters)
commit_hash=$(git rev-parse --short=5 HEAD)
# Combine branch and commit hash
buildName="${USER}-${branch}-${commit_hash}"
buildName="${USER}-${cleaned_branch}-${commit_hash}"
gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${buildName}