Merge branch 'RED-7883' into 'master'

Resolve RED-7883

Closes RED-7883

See merge request redactmanager/persistence-service!211

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java

@@ -3,12 +3,15 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
 import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;
 
 import java.io.BufferedInputStream;
+import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
@@ -17,7 +20,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.function.Function;
@@ -67,7 +69,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ReportTemplateUploadRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ExportFilename;
@@ -531,19 +532,12 @@ public class DossierTemplateImportService {
     private List<String> readEntries(byte[] bytes) {
 
         ByteArrayInputStream bInput = new ByteArrayInputStream(bytes);
-        DataInputStream in = new DataInputStream(bInput);
-        List<String> entries = new ArrayList<>();
-        try {
-            while (in.available() > 0) {
-                String entry = in.readUTF();
-                entries.add(entry);
-
-            }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(bInput, StandardCharsets.UTF_8))) {
+            return br.lines().collect(Collectors.toList());
         } catch (IOException e) {
             log.debug("exception: ", e);
             throw new BadRequestException("Error while reading the entries", e);
         }
-        return entries;
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java

@@ -1,18 +1,5 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.export;
 
-import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
@@ -26,16 +13,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJ
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -54,11 +32,19 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
-
-import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;
 
 @Slf4j
 @Service
@@ -254,15 +240,23 @@ public class DossierTemplateExportService {
 
     private void writeEntriesListToFile(FileSystemBackedArchiver fileSystemBackedArchiver, List<String> entriesValuesList, String folderName, String filename) {
 
-        try {
-            ByteArrayOutputStream bt = new ByteArrayOutputStream();
-            DataOutputStream dt = new DataOutputStream(bt);
-            for (String entry : entriesValuesList) {
-                dt.writeUTF(entry);
+        try (ByteArrayOutputStream bt = new ByteArrayOutputStream();
+             OutputStreamWriter osw = new OutputStreamWriter(bt, StandardCharsets.UTF_8);
+             BufferedWriter writer = new BufferedWriter(osw)) {
+
+            Iterator<String> iterator = entriesValuesList.iterator();
+
+            while (iterator.hasNext()) {
+                String entry = iterator.next();
+                writer.write(entry);
+
+                if (iterator.hasNext()) {
+                    writer.newLine();
+                }
             }
+            writer.flush();
+
             fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folderName, filename, bt.toByteArray()));
-            dt.close();
-            bt.close();
         } catch (IOException e) {
             log.debug("Error writing values to files");
         }

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportTest.java

@@ -47,7 +47,7 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT
     @Disabled
     public void testLocalDossierTemplateImport() {
 
-        var importDir = new File("C:\\Users\\YannikHampe\\Downloads\\testing\\");
+        var importDir = new File("/tmp/knoell");
         assertThat(importDir).isNotNull();
         assertThat(importDir.exists()).isTrue();
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java

@@ -1,50 +1,18 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.InputStream;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
-import java.time.temporal.ChronoUnit;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.jetbrains.annotations.NotNull;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.mock.web.MockMultipartFile;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Sets;
-import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.DossierAttributeConfigClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.DossierStatusClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.LegalBasisClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.ReportTemplateClient;
-import com.iqser.red.service.peristence.v1.server.integration.client.WatermarkClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.*;
 import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.management.v1.processor.service.export.ExportDownloadMessageReceiver;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierAttributesConfig;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStatusRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.*;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
@@ -59,9 +27,33 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
-
 import feign.FeignException;
 import lombok.SneakyThrows;
+import org.jetbrains.annotations.NotNull;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.mock.web.MockMultipartFile;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
 
@@ -551,6 +543,117 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    public void testWriteLines() {
+
+        List<String> entriesValuesList = List.of("Adam", "Eva", "drei sdfs");
+
+        try (ByteArrayOutputStream bt = new ByteArrayOutputStream();
+             OutputStreamWriter osw = new OutputStreamWriter(bt, StandardCharsets.UTF_8);
+             BufferedWriter writer = new BufferedWriter(osw)) {
+
+            for (String entry : entriesValuesList) {
+                writer.write(entry);
+                writer.newLine();
+            }
+            writer.flush();
+
+            try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/e.txt")) {
+                fileOutputStream.write(bt.toByteArray());
+            }
+
+        }
+    }
+
+    @Test
+    @Disabled
+    @SneakyThrows
+    public void changeEncodingFromModUTF8ToUTF8() {
+        String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/dev-v2/";
+        Stream.of(
+                        Files.walk(Path.of(dossierTemplatesRepo + "EFSA_Regulation_2021")),
+                        Files.walk(Path.of(dossierTemplatesRepo + "EFSA_sanitisation_GFL_v1")),
+                        Files.walk(Path.of(dossierTemplatesRepo + "EFSA_sanitisation_pre_GFL_v1")),
+                        Files.walk(Path.of(dossierTemplatesRepo + "Flora_SCM_(Syngenta_Evaluation)")),
+                        Files.walk(Path.of(dossierTemplatesRepo + "Manual_Redaction"))//
+                )
+                .flatMap(Function.identity())//
+                .filter(path -> path.getFileName().toString().equals("entries.txt") || //
+                        path.getFileName().toString().equals("falsePositives.txt") || //
+                        path.getFileName().toString().equals("falseRecommendations.txt")//
+                )
+                .map(Path::toFile)//
+                .forEach(file -> {
+                    System.out.println(file);
+                    try {
+                        changeEncoding(file);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+    }
+
+    private void changeEncoding(File modUTF8File) throws IOException {
+        var path = modUTF8File.getPath();
+
+        var bytes = convertFileToByteArray(modUTF8File);
+
+        var entries = readEntriesModUTF8(bytes);
+        writeEntriesListToFileUTF8(entries, path);
+
+    }
+
+    private static byte[] convertFileToByteArray(File file) throws IOException {
+        Path path = file.toPath();
+        return Files.readAllBytes(path);
+    }
+
+
+    private List<String> readEntriesModUTF8(byte[] bytes) {
+
+        ByteArrayInputStream bInput = new ByteArrayInputStream(bytes);
+        DataInputStream in = new DataInputStream(bInput);
+        List<String> entries = new ArrayList<>();
+        try {
+            while (in.available() > 0) {
+                String entry = in.readUTF();
+                entries.add(entry);
+            }
+        } catch (IOException e) {
+            throw new BadRequestException("Error while reading the entries", e);
+        }
+        return entries;
+    }
+
+    private void writeEntriesListToFileUTF8(List<String> entriesValuesList, String path) {
+
+        try (ByteArrayOutputStream bt = new ByteArrayOutputStream();
+             OutputStreamWriter osw = new OutputStreamWriter(bt, StandardCharsets.UTF_8);
+             BufferedWriter writer = new BufferedWriter(osw);
+             FileOutputStream fileOutputStream = new FileOutputStream(path)) {
+
+            Iterator<String> iterator = entriesValuesList.iterator();
+
+            while (iterator.hasNext()) {
+                String entry = iterator.next();
+                writer.write(entry);
+
+                if (iterator.hasNext()) {
+                    writer.newLine();
+                }
+            }
+            writer.flush();
+
+            bt.writeTo(fileOutputStream);
+            System.out.println("Data written to file successfully!");
+
+        } catch (IOException e) {
+            System.out.println("Error writing values to files");
+        }
+    }
+
+
     @Test
     @SneakyThrows
     public void testImportDossierTemplateUpdateExisting() {