diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java index 17a5ed46f..10ba702d7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateImportService.java @@ -3,12 +3,15 @@ package com.iqser.red.service.persistence.management.v1.processor.service; import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert; import java.io.BufferedInputStream; +import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -17,7 +20,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.Function; @@ -67,7 +69,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ReportTemplateUploadRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ExportFilename; @@ -531,19 +532,12 @@ public class DossierTemplateImportService { private List readEntries(byte[] bytes) { ByteArrayInputStream bInput = new ByteArrayInputStream(bytes); - DataInputStream in = new DataInputStream(bInput); - List entries = new ArrayList<>(); - try { - while (in.available() > 0) { - String entry = in.readUTF(); - entries.add(entry); - - } + try (BufferedReader br = new BufferedReader(new InputStreamReader(bInput, StandardCharsets.UTF_8))) { + return br.lines().collect(Collectors.toList()); } catch (IOException e) { log.debug("exception: ", e); throw new BadRequestException("Error while reading the entries", e); } - return entries; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java index b8f288cde..385bcd56e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java @@ -1,18 +1,5 @@ package com.iqser.red.service.persistence.management.v1.processor.service.export; -import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert; - -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.stream.Collectors; - -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; @@ -26,16 +13,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJ import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver; import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; @@ -54,11 +32,19 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; - -import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert; @Slf4j @Service @@ -254,15 +240,23 @@ public class DossierTemplateExportService { private void writeEntriesListToFile(FileSystemBackedArchiver fileSystemBackedArchiver, List entriesValuesList, String folderName, String filename) { - try { - ByteArrayOutputStream bt = new ByteArrayOutputStream(); - DataOutputStream dt = new DataOutputStream(bt); - for (String entry : entriesValuesList) { - dt.writeUTF(entry); + try (ByteArrayOutputStream bt = new ByteArrayOutputStream(); + OutputStreamWriter osw = new OutputStreamWriter(bt, StandardCharsets.UTF_8); + BufferedWriter writer = new BufferedWriter(osw)) { + + Iterator iterator = entriesValuesList.iterator(); + + while (iterator.hasNext()) { + String entry = iterator.next(); + writer.write(entry); + + if (iterator.hasNext()) { + writer.newLine(); + } } + writer.flush(); + fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folderName, filename, bt.toByteArray())); - dt.close(); - bt.close(); } catch (IOException e) { log.debug("Error writing values to files"); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportTest.java index 3bf948eb5..b4b82ff22 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportTest.java @@ -47,7 +47,7 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT @Disabled public void testLocalDossierTemplateImport() { - var importDir = new File("C:\\Users\\YannikHampe\\Downloads\\testing\\"); + var importDir = new File("/tmp/knoell"); assertThat(importDir).isNotNull(); assertThat(importDir.exists()).isTrue(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index 01a0af423..f102991f5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -1,50 +1,18 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.InputStream; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.Test; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.mock.web.MockMultipartFile; - import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; -import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierAttributeConfigClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierStatusClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient; -import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient; -import com.iqser.red.service.peristence.v1.server.integration.client.LegalBasisClient; -import com.iqser.red.service.peristence.v1.server.integration.client.ReportTemplateClient; -import com.iqser.red.service.peristence.v1.server.integration.client.WatermarkClient; +import com.iqser.red.service.peristence.v1.server.integration.client.*; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.export.ExportDownloadMessageReceiver; -import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue; -import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierAttributesConfig; -import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStatusRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig; -import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue; -import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.*; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus; @@ -59,9 +27,33 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; - import feign.FeignException; import lombok.SneakyThrows; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.mock.web.MockMultipartFile; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { @@ -551,6 +543,117 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { } + @Test + @SneakyThrows + public void testWriteLines() { + + List entriesValuesList = List.of("Adam", "Eva", "drei sdfs"); + + try (ByteArrayOutputStream bt = new ByteArrayOutputStream(); + OutputStreamWriter osw = new OutputStreamWriter(bt, StandardCharsets.UTF_8); + BufferedWriter writer = new BufferedWriter(osw)) { + + for (String entry : entriesValuesList) { + writer.write(entry); + writer.newLine(); + } + writer.flush(); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/e.txt")) { + fileOutputStream.write(bt.toByteArray()); + } + + } + } + + @Test + @Disabled + @SneakyThrows + public void changeEncodingFromModUTF8ToUTF8() { + String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/dev-v2/"; + Stream.of( + Files.walk(Path.of(dossierTemplatesRepo + "EFSA_Regulation_2021")), + Files.walk(Path.of(dossierTemplatesRepo + "EFSA_sanitisation_GFL_v1")), + Files.walk(Path.of(dossierTemplatesRepo + "EFSA_sanitisation_pre_GFL_v1")), + Files.walk(Path.of(dossierTemplatesRepo + "Flora_SCM_(Syngenta_Evaluation)")), + Files.walk(Path.of(dossierTemplatesRepo + "Manual_Redaction"))// + ) + .flatMap(Function.identity())// + .filter(path -> path.getFileName().toString().equals("entries.txt") || // + path.getFileName().toString().equals("falsePositives.txt") || // + path.getFileName().toString().equals("falseRecommendations.txt")// + ) + .map(Path::toFile)// + .forEach(file -> { + System.out.println(file); + try { + changeEncoding(file); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private void changeEncoding(File modUTF8File) throws IOException { + var path = modUTF8File.getPath(); + + var bytes = convertFileToByteArray(modUTF8File); + + var entries = readEntriesModUTF8(bytes); + writeEntriesListToFileUTF8(entries, path); + + } + + private static byte[] convertFileToByteArray(File file) throws IOException { + Path path = file.toPath(); + return Files.readAllBytes(path); + } + + + private List readEntriesModUTF8(byte[] bytes) { + + ByteArrayInputStream bInput = new ByteArrayInputStream(bytes); + DataInputStream in = new DataInputStream(bInput); + List entries = new ArrayList<>(); + try { + while (in.available() > 0) { + String entry = in.readUTF(); + entries.add(entry); + } + } catch (IOException e) { + throw new BadRequestException("Error while reading the entries", e); + } + return entries; + } + + private void writeEntriesListToFileUTF8(List entriesValuesList, String path) { + + try (ByteArrayOutputStream bt = new ByteArrayOutputStream(); + OutputStreamWriter osw = new OutputStreamWriter(bt, StandardCharsets.UTF_8); + BufferedWriter writer = new BufferedWriter(osw); + FileOutputStream fileOutputStream = new FileOutputStream(path)) { + + Iterator iterator = entriesValuesList.iterator(); + + while (iterator.hasNext()) { + String entry = iterator.next(); + writer.write(entry); + + if (iterator.hasNext()) { + writer.newLine(); + } + } + writer.flush(); + + bt.writeTo(fileOutputStream); + System.out.println("Data written to file successfully!"); + + } catch (IOException e) { + System.out.println("Error writing values to files"); + } + } + + @Test @SneakyThrows public void testImportDossierTemplateUpdateExisting() { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/EFSA_sanitisation_GFL_v1.zip b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/EFSA_sanitisation_GFL_v1.zip index 1346866d7..ea870134e 100644 Binary files a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/EFSA_sanitisation_GFL_v1.zip and b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/EFSA_sanitisation_GFL_v1.zip differ diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/EFSA_sanitisation_GFL_v1_adress_parts.zip b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/EFSA_sanitisation_GFL_v1_adress_parts.zip index b11ca6177..492219dfc 100644 Binary files a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/EFSA_sanitisation_GFL_v1_adress_parts.zip and b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/dossiertemplates/EFSA_sanitisation_GFL_v1_adress_parts.zip differ