Merge branch 'feature/RED-10260-bp' into 'release/2.589.x'

Resolve RED-10260 "Feature/ bp"

See merge request redactmanager/persistence-service!795

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java

@@ -269,7 +269,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     @Override
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
-    public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter) {
+    public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter, String quoteChar) {
 
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
@@ -285,18 +285,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
             throw new BadRequestException(format("The provided file name \"%s\" is not valid!", nameToUse));
         }
 
-        if (Strings.isNullOrEmpty(delimiter)) {
+        char cleanDelimiter = getDelimiter(delimiter);
-            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
+        char cleanQuoteChar = getQuoteChar(quoteChar);
-        } else if (delimiter.length() != 1) {
-            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
-        }
-        char cleanDelimiter = delimiter.charAt(0);
 
         Path mappingFile = saveToFile(file);
 
         try {
 
-            ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, cleanDelimiter, encoding, mappingFile.toFile());
+            ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
+                                                                               nameToUse,
+                                                                               fileName,
+                                                                               cleanDelimiter,
+                                                                               encoding,
+                                                                               mappingFile.toFile(),
+                                                                               cleanQuoteChar);
 
             return componentMappingMapper.toModel(metaData);
         } finally {
@@ -309,18 +311,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     @Override
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
-    public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String name, String encoding, String delimiter) {
+    public ComponentMappingMetadataModel updateMapping(String dossierTemplateId,
+                                                       String componentMappingId,
+                                                       MultipartFile file,
+                                                       String name,
+                                                       String encoding,
+                                                       String delimiter,
+                                                       String quoteChar) {
 
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
         String nameToUse = validateFileName(file, name);
 
-        if (Strings.isNullOrEmpty(delimiter)) {
+        char cleanDelimiter = getDelimiter(delimiter);
-            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
+        char cleanQuoteChar = getQuoteChar(quoteChar);
-        } else if (delimiter.length() != 1) {
-            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
-        }
-        char cleanDelimiter = delimiter.charAt(0);
 
         Path mappingFile = saveToFile(file);
 
@@ -331,7 +335,8 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
                                                                                      encoding,
                                                                                      cleanDelimiter,
                                                                                      mappingFile.toFile(),
-                                                                                     file.getOriginalFilename());
+                                                                                     file.getOriginalFilename(),
+                                                                                     cleanQuoteChar);
 
             return componentMappingMapper.toModel(resultMetaData);
         } finally {
@@ -340,6 +345,28 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     }
 
 
+    private static char getDelimiter(String delimiter) {
+
+        if (Strings.isNullOrEmpty(delimiter)) {
+            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
+        } else if (delimiter.length() != 1) {
+            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
+        }
+        return delimiter.charAt(0);
+    }
+
+
+    private static char getQuoteChar(String quoteChar) {
+
+        if (Strings.isNullOrEmpty(quoteChar)) {
+            throw new BadRequestException("The provided quoteChar is not valid! Can't be null or empty.");
+        } else if (quoteChar.length() != 1) {
+            throw new BadRequestException(format("The provided quoteChar %s is not valid! Only a single character is allowed.", quoteChar));
+        }
+        return quoteChar.charAt(0);
+    }
+
+
     private static String validateFileName(MultipartFile file, String name) {
 
         if (Strings.isNullOrEmpty(file.getOriginalFilename()) || !file.getOriginalFilename().endsWith(".csv")) {

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.service.v2.api.external.resource;
 import java.util.List;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DateFormatPatternErrorMessage;
 import com.iqser.red.service.persistence.service.v2.api.external.model.DossierAttributeDefinitionList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.DossierStatusDefinitionList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList;
@@ -63,6 +62,7 @@ public interface DossierTemplateResource {
     String DRY_RUN_PARAM = "dryRun";
     String ENCODING_PARAM = "encoding";
     String DELIMITER_PARAM = "delimiter";
+    String QUOTE_CHAR_PARAM = "quoteChar";
     String MAPPING_NAME_PARAM = "name";
 
     String INCLUDE_SOFT_DELETED = "includeSoftDeleted";
@@ -127,7 +127,7 @@ public interface DossierTemplateResource {
     @Operation(summary = "Upload a date formats file for a specific DossierTemplate.")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Date formats upload successful."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found."), @ApiResponse(responseCode = "400", description = "Uploaded date formats could not be verified."), @ApiResponse(responseCode = "422", description = "Uploaded date formats file could not be parsed.")})
     ResponseEntity<?> uploadDateFormats(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
-                                                                          @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
+                                        @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
 
     @ResponseBody
@@ -159,7 +159,8 @@ public interface DossierTemplateResource {
                                                 @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                 @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
                                                 @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
-                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
+                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
+                                                @Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
 
 
     @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None")
@@ -173,7 +174,8 @@ public interface DossierTemplateResource {
                                                 @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                 @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
                                                 @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
-                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
+                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
+                                                @Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
 
 
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml

@@ -509,7 +509,7 @@ paths:
         - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
         
         #### Customization Options
-        - Users can specify the delimiter and encoding used in the CSV file.
+        - Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
         
         #### Usage
         - The component mapping file can be utilized in component rules to relate components to existing master data.
@@ -533,6 +533,7 @@ paths:
         - $ref: '#/components/parameters/mappingName'
         - $ref: '#/components/parameters/encoding'
         - $ref: '#/components/parameters/delimiter'
+        - $ref: '#/components/parameters/quoteChar'
       responses:
         "200":
           content:
@@ -609,7 +610,7 @@ paths:
         - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
         
         #### Customization Options
-        - Users can specify the delimiter and encoding used in the CSV file.
+        - Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
       tags:
         - 1. Dossier Templates
       requestBody:
@@ -623,6 +624,7 @@ paths:
         - $ref: '#/components/parameters/mappingName'
         - $ref: '#/components/parameters/encoding'
         - $ref: '#/components/parameters/delimiter'
+        - $ref: '#/components/parameters/quoteChar'
       responses:
         "200":
           content:
@@ -2114,6 +2116,17 @@ components:
         example: ','
         default: ','
       description: "The delimiter used as a separator in a csv file."
+    quoteChar:
+      name: quoteChar
+      required: false
+      in: query
+      schema:
+        type: string
+        minLength: 1
+        maxLength: 1
+        example: '"'
+        default: '"'
+      description: "The quoteChar used to quote fields in a csv file."
     mappingName:
       name: name
       required: false

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateImportService.java

@@ -75,10 +75,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -449,7 +447,8 @@ public class DossierTemplateImportService {
                                                                                      componentMapping.metadata().getFileName(),
                                                                                      componentMapping.metadata().getDelimiter(),
                                                                                      componentMapping.metadata().getEncoding(),
-                                                                                     tmpFile);
+                                                                                     tmpFile,
+                                                                                     componentMapping.metadata().getQuoteChar());
 
             componentMappingService.setVersion(createdMapping.getId(), componentMapping.metadata().getVersion());
             assert tmpFile.delete();
@@ -524,6 +523,7 @@ public class DossierTemplateImportService {
 
     }
 
+
     private LayoutParsingType deferFromApplicationType() {
 
         return Objects.equals(applicationType, "DocuMine") ? LayoutParsingType.DOCUMINE_OLD : LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java

@@ -75,4 +75,7 @@ public class ComponentMappingEntity {
     @Builder.Default
     char delimiter = ',';
 
+    @Builder.Default
+    char quoteChar = '"';
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java

@@ -62,16 +62,23 @@ public class ComponentMappingService {
 
 
     @SneakyThrows
-    public ComponentMappingMetadata update(String dossierTemplateId, String mappingId, String name, String encoding, char delimiter, File mappingFile, String fileName) {
+    public ComponentMappingMetadata update(String dossierTemplateId,
+                                           String mappingId,
+                                           String name,
+                                           String encoding,
+                                           char delimiter,
+                                           File mappingFile,
+                                           String fileName,
+                                           char quoteChar) {
 
         ComponentMappingEntity entity = componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId);
 
-        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName);
+        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName, quoteChar);
     }
 
 
     @SneakyThrows
-    public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile) {
+    public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile, char quoteChar) {
 
         if (componentMappingPersistenceService.existsByNameAndDossierTemplateId(name, dossierTemplateId)) {
             throw new BadRequestException("A mapping with this name already exists in the dossier template!");
@@ -86,20 +93,27 @@ public class ComponentMappingService {
                 .fileName(fileName)
                 .build();
 
-        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName);
+        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName, quoteChar);
 
     }
 
 
     @SneakyThrows
-    private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity, String name, String encoding, char delimiter, File mappingFile, String fileName) {
+    private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity,
+                                                    String name,
+                                                    String encoding,
+                                                    char delimiter,
+                                                    File mappingFile,
+                                                    String fileName,
+                                                    char quoteChar) {
 
         Charset charset = resolveCharset(encoding);
 
-        CsvStats stats = sortCSVFile(delimiter, mappingFile, charset);
+        CsvStats stats = sortCSVFile(delimiter, mappingFile, charset, quoteChar);
 
         entity.setName(name);
         entity.setDelimiter(delimiter);
+        entity.setQuoteChar(quoteChar);
         entity.setEncoding(encoding);
         entity.setNumberOfLines(stats.numberOfLines());
         entity.setColumnLabels(stats.columnLabels());
@@ -126,7 +140,7 @@ public class ComponentMappingService {
     }
 
 
-    private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws BadRequestException, IOException {
+    private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset, char quoteChar) throws BadRequestException, IOException {
 
         Path tempFile = Files.createTempFile("mapping", ".tmp");
 
@@ -135,11 +149,8 @@ public class ComponentMappingService {
         String[] columnLabels;
         int numberOfLines = 0;
         try (Reader fileReader = new FileReader(tempFile.toFile(), charset);//
-             CSVReader reader = buildReader(fileReader, delimiter);//
+             CSVReader reader = buildReader(fileReader, delimiter, quoteChar);//
-             CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset), delimiter,
+             CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset), delimiter, quoteChar, '\\', CSVWriter.DEFAULT_LINE_END)) {
-                                              CSVWriter.NO_QUOTE_CHARACTER,
-                                              CSVWriter.DEFAULT_ESCAPE_CHARACTER,
-                                              CSVWriter.DEFAULT_LINE_END)) {
 
             List<String[]> rows = reader.readAll();
 
@@ -180,9 +191,9 @@ public class ComponentMappingService {
     }
 
 
-    private static CSVReader buildReader(Reader reader, char delimiter) throws IOException {
+    private static CSVReader buildReader(Reader reader, char delimiter, char quoteChar) throws IOException {
 
-        return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build();
+        return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).withQuoteChar(quoteChar).build()).build();
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java

@@ -216,7 +216,8 @@ public class DossierTemplateCloneService {
                                            componentMapping.metaData().getFileName(),
                                            componentMapping.metaData().getDelimiter(),
                                            componentMapping.metaData().getEncoding(),
-                                           componentMapping.file());
+                                           componentMapping.file(),
+                                           componentMapping.metaData().getQuoteChar());
         }
         FileSystemUtils.deleteRecursively(dir);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml

@@ -235,3 +235,5 @@ databaseChangeLog:
       file: db/changelog/tenant/145-add-indexes-to-file-table.yaml
   - include:
       file: db/changelog/tenant/146-add-layout-parsing-type-to-dossier-template.yaml
+  - include:
+      file: db/changelog/tenant/147-add-quotechar-to-component-mapping.yaml

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/147-add-quotechar-to-component-mapping.yaml

@@ -0,0 +1,14 @@
+databaseChangeLog:
+  - changeSet:
+      id: add-quote_char-to-component-mapping
+      author: kilian
+      changes:
+        - addColumn:
+            tableName: component_mappings
+            columns:
+              - column:
+                  name: quote_char
+                  type: char
+                  defaultValue: '"'
+                  constraints:
+                    nullable: false

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentMappingTest.java

@@ -53,7 +53,12 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
                                                       "application/csv",
                                                       IOUtils.toByteArray(new ClassPathResource("files/componentmapping/file.csv").getInputStream()));
 
-        ComponentMappingMetadataModel componentMappingMetadataModel = dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(), mockMultipartFile, "file", "UTF-8", ",");
+        ComponentMappingMetadataModel componentMappingMetadataModel = dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(),
+                                                                                                                  mockMultipartFile,
+                                                                                                                  "file",
+                                                                                                                  "UTF-8",
+                                                                                                                  ",",
+                                                                                                                  "\"");
 
         assertEquals(componentMappingMetadataModel.getFileName(), mockMultipartFile.getOriginalFilename());
 
@@ -65,11 +70,20 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
         when(componentMappingPersistenceService.getEntityById(anyString(), anyString())).thenReturn(ComponentMappingEntity.builder()
                                                                                                             .id(componentMappingMetadataModel.getId())
                                                                                                             .dossierTemplate(new DossierTemplateEntity())
-                                                                                                            .storageId(buildStorageId(dossierTemplate.getId(), componentMappingMetadataModel.getId(), "file", "update_file.csv"))
+                                                                                                            .storageId(buildStorageId(dossierTemplate.getId(),
+                                                                                                                                      componentMappingMetadataModel.getId(),
+                                                                                                                                      "file",
+                                                                                                                                      "update_file.csv"))
                                                                                                             .fileName("update_file.csv")
                                                                                                             .build());
 
-        componentMappingMetadataModel = dossierTemplateExternalClient.updateMapping(dossierTemplate.getId(), componentMappingMetadataModel.getId(), updateMockMultipartFile, "file", "UTF-8", ",");
+        componentMappingMetadataModel = dossierTemplateExternalClient.updateMapping(dossierTemplate.getId(),
+                                                                                    componentMappingMetadataModel.getId(),
+                                                                                    updateMockMultipartFile,
+                                                                                    "file",
+                                                                                    "UTF-8",
+                                                                                    ",",
+                                                                                    "\"");
 
         assertEquals(componentMappingMetadataModel.getFileName(), updateMockMultipartFile.getOriginalFilename());
     }
@@ -86,7 +100,8 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
                                                       "application/csv",
                                                       IOUtils.toByteArray(new ClassPathResource("files/componentmapping/empty.csv").getInputStream()));
 
-        var result = assertThrows(FeignException.class, () -> dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(), mockMultipartFile, "file", "UTF-8", ","));
+        String id = dossierTemplate.getId();
+        var result = assertThrows(FeignException.class, () -> dossierTemplateExternalClient.uploadMapping(id, mockMultipartFile, "file", "UTF-8", ",", "\""));
         assertTrue(result.getMessage().contains("CSV file can not be empty!"));
     }
 
@@ -95,4 +110,5 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
 
         return dossierTemplateId + "/" + id + "_" + name + "_" + fileName;
     }
+
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java

@@ -35,4 +35,6 @@ public class ComponentMappingMetadata {
 
     char delimiter;
 
+    char quoteChar;
+
 }

publish-custom-image.sh

@@ -9,11 +9,14 @@ gradle assemble
 # Get the current Git branch
 branch=$(git rev-parse --abbrev-ref HEAD)
 
+# Replace any slashes (e.g., in 'feature/' or 'release/') with a hyphen
+cleaned_branch=$(echo "$branch" | sed 's/\//_/g')
+
 # Get the short commit hash (first 5 characters)
 commit_hash=$(git rev-parse --short=5 HEAD)
 
 # Combine branch and commit hash
-buildName="${USER}-${branch}-${commit_hash}"
+buildName="${USER}-${cleaned_branch}-${commit_hash}"
 
 gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${buildName}