Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-7883: changed encoding of type dictionaries from modified UTF-8 to UTF-8

Browse Source
This commit is contained in:
Ali Oezyetimoglu 2023-11-14 10:44:21 +01:00
parent d11a1178b3
commit f0a4772c33
1 changed files with 122 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java
View File

@ -1,59 +1,18 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Test;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierAttributeConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierStatusClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.LegalBasisClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ReportTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.client.WatermarkClient;
import com.iqser.red.service.peristence.v1.server.integration.client.*;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.export.ExportDownloadMessageReceiver;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierAttributesConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStatusRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.*;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
@ -68,9 +27,33 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import feign.FeignException;
import lombok.SneakyThrows;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
@ -122,6 +105,11 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
@Autowired
private LegalBasisClient legalBasisClient;
@Autowired
private DossierTemplateImportService dossierTemplateImportService;
@Autowired
private DossierTemplateManagementService dossierTemplateManagementService;
@Test
public void testDownload() {
@ -385,7 +373,6 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
}
private void setupDossierTemplate(DossierTemplateModel dossierTemplate) {
var type = CreateTypeValue.builder()
@ -584,6 +571,94 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
}
}
@Test
@Disabled
@SneakyThrows
public void changeEncodingFromModUTF8ToUTF8() {
String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/dev-v2/";
Stream.of(
Files.walk(Path.of(dossierTemplatesRepo + "EFSA_Regulation_2021")),
Files.walk(Path.of(dossierTemplatesRepo + "EFSA_sanitisation_GFL_v1")),
Files.walk(Path.of(dossierTemplatesRepo + "EFSA_sanitisation_pre_GFL_v1")),
Files.walk(Path.of(dossierTemplatesRepo + "Flora_SCM_(Syngenta_Evaluation)")),
Files.walk(Path.of(dossierTemplatesRepo + "Manual_Redaction"))//
)
.flatMap(Function.identity())//
.filter(path -> path.getFileName().toString().equals("entries.txt") || //
path.getFileName().toString().equals("falsePositives.txt") || //
path.getFileName().toString().equals("falseRecommendations.txt")//
)
.map(Path::toFile)//
.forEach(file -> {
System.out.println(file);
try {
changeEncoding(file);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
private void changeEncoding(File modUTF8File) throws IOException {
var path = modUTF8File.getPath();
var bytes = convertFileToByteArray(modUTF8File);
var entries = readEntriesModUTF8(bytes);
writeEntriesListToFileUTF8(entries, path);
}
private static byte[] convertFileToByteArray(File file) throws IOException {
Path path = file.toPath();
return Files.readAllBytes(path);
}
private List<String> readEntriesModUTF8(byte[] bytes) {
ByteArrayInputStream bInput = new ByteArrayInputStream(bytes);
DataInputStream in = new DataInputStream(bInput);
List<String> entries = new ArrayList<>();
try {
while (in.available() > 0) {
String entry = in.readUTF();
entries.add(entry);
}
} catch (IOException e) {
throw new BadRequestException("Error while reading the entries", e);
}
return entries;
}
private void writeEntriesListToFileUTF8(List<String> entriesValuesList, String path) {
try (ByteArrayOutputStream bt = new ByteArrayOutputStream();
OutputStreamWriter osw = new OutputStreamWriter(bt, StandardCharsets.UTF_8);
BufferedWriter writer = new BufferedWriter(osw);
FileOutputStream fileOutputStream = new FileOutputStream(path)) {
Iterator<String> iterator = entriesValuesList.iterator();
while (iterator.hasNext()) {
String entry = iterator.next();
writer.write(entry);
if (iterator.hasNext()) {
writer.newLine();
}
}
writer.flush();
bt.writeTo(fileOutputStream);
System.out.println("Data written to file successfully!");
} catch (IOException e) {
System.out.println("Error writing values to files");
}
}
@Test
@SneakyThrows