Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

Pull request #228: RED-6603 1

Browse Source 
Merge in RED/redaction-report-service from RED-6603_1 to master

* commit '4c2b3a779646f826585c5824fcbe3dd962bd2922':
  RED-6603 - change Assertions.assertIterableEquals() to assertThat().isEqualTo()
  RED-6603 - use paragraphs for line-separation and adjust test
  RED-6603 - add test for iuclid
This commit is contained in:
Thomas Beyer 2023-05-05 10:43:29 +02:00
commit 24d7de1cf5
3 changed files with 164 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java
View File

@ -31,6 +31,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
@ -225,11 +226,11 @@ public class WordReportGenerationService {
private int replaceTextPlaceholders(XWPFDocument doc,
PlaceholderModel placeholderModel,
String dossierName,
String fileName,
XWPFTable tableToSkip,
List<ReportRedactionEntry> reportRedactionEntries) {
PlaceholderModel placeholderModel,
String dossierName,
String fileName,
XWPFTable tableToSkip,
List<ReportRedactionEntry> reportRedactionEntries) {
int sumOfChars = 0;
@ -239,7 +240,7 @@ public class WordReportGenerationService {
placeHolderValueMap.put(placeholder, placeholderValue);
}
replacePlaceholderInParagraph(doc.getParagraphs(), placeHolderValueMap);
replacePlaceholderInParagraphAndSplit(doc.getParagraphs(), placeHolderValueMap, doc);
for (XWPFTable tbl : doc.getTables()) {
if (tableToSkip == tbl) {
// already processed for redactionLog Entries
@ -247,7 +248,7 @@ public class WordReportGenerationService {
}
for (XWPFTableRow row : tbl.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
sumOfChars += replacePlaceholderInParagraph(cell.getParagraphs(), placeHolderValueMap);
sumOfChars += replacePlaceholderInParagraphAndSplit(cell.getParagraphs(), placeHolderValueMap, doc);
}
}
}
@ -256,11 +257,14 @@ public class WordReportGenerationService {
}
private int replacePlaceholderInParagraph(List<XWPFParagraph> paragraphs, Map<String, String> placeholderValueMap) {
private int replacePlaceholderInParagraphAndSplit(List<XWPFParagraph> paragraphs, Map<String, String> placeholderValueMap, XWPFDocument doc) {
int sumOfChars = 0;
for (XWPFParagraph p : paragraphs) {
List<XWPFParagraph> newLineParagraphs = new ArrayList<>();
boolean isIuclid = false;
int paraSize = paragraphs.size();
for (int paraIndex = 0; paraIndex < paraSize; paraIndex++) {
XWPFParagraph p = paragraphs.get(paraIndex);
String paragraphText = p.getText();
for (var entry : placeholderValueMap.entrySet()) {
@ -269,6 +273,7 @@ public class WordReportGenerationService {
var replace = entry.getValue();
if (paragraphText.contains(search)) {
isIuclid = search.equals(IUCLID_FUNCTION_PLACEHOLDER);
String escapedSearch = Pattern.quote(search);
String escapedReplace = Matcher.quoteReplacement(replace);
try {
@ -280,36 +285,86 @@ public class WordReportGenerationService {
log.error("Could not replace {} with {}", escapedSearch, escapedReplace);
throw new RuntimeException(String.format("Could not replace %s with %s", escapedSearch, escapedReplace), e);
}
int size = p.getRuns().size();
for (int i = 1; i <= size; i++) {
int runSize = p.getRuns().size();
for (int runIdx = 1; runIdx <= runSize; runIdx++) {
// 0th run has to stay for placeholders without "\n", because it contains the value of the placeholder
p.removeRun(1);
}
if (paragraphText.contains("\n")) {
p.removeRun(0);
String[] stringsOnNewLines = paragraphText.split("\n");
for (int i = 0; i < stringsOnNewLines.length; i++) {
p.insertNewRun(i);
XWPFRun newRun = p.getRuns().get(i);
String textForLine = stringsOnNewLines[i];
ColoredText coloredText = getColor(textForLine);
newRun.setText(coloredText.getText());
sumOfChars += coloredText.getText().length();
if (coloredText.getColor() != null) {
newRun.setTextHighlightColor(coloredText.getColor());
}
newRun.addCarriageReturn();
}
split(doc, newLineParagraphs, isIuclid, p, paragraphText);
}
}
}
}
if (isIuclid) {
replaceParagraphsAndRemoveRedundantOnes(doc, newLineParagraphs);
}
return sumOfChars;
}
private void split(XWPFDocument doc, List<XWPFParagraph> newLineParagraphs, boolean isIuclid, XWPFParagraph p, String paragraphText) {
p.removeRun(0);
String[] stringsOnNewLines = paragraphText.split("\n");
for (int i = 0; i < stringsOnNewLines.length; i++) {
if (isIuclid) {
splitIntoParagraphs(doc, newLineParagraphs, stringsOnNewLines[i]);
} else {
splitIntoRuns(p, stringsOnNewLines, i);
}
}
}
private void splitIntoRuns(XWPFParagraph p, String[] stringsOnNewLines, int i) {
p.insertNewRun(i);
XWPFRun newRun = p.getRuns().get(i);
setTextToRun(newRun, stringsOnNewLines[i]);
newRun.addCarriageReturn();
}
private void splitIntoParagraphs(XWPFDocument doc, List<XWPFParagraph> newLineParagraphs, String stringsOnNewLines) {
XWPFParagraph newParagraph = doc.createParagraph();
XWPFRun newRun = newParagraph.createRun();
setTextToRun(newRun, stringsOnNewLines);
newLineParagraphs.add(newParagraph);
}
private void replaceParagraphsAndRemoveRedundantOnes(XWPFDocument doc, List<XWPFParagraph> newLineParagraphs) {
List<XWPFParagraph> paragraphsToRemove = new ArrayList<>();
for (int i = 0; i < doc.getParagraphs().size(); i++) {
if (i < newLineParagraphs.size()) {
doc.setParagraph(newLineParagraphs.get(i), i);
} else {
paragraphsToRemove.add(doc.getParagraphs().get(i));
}
}
for (XWPFParagraph paragraphToRemove : paragraphsToRemove) {
doc.removeBodyElement(doc.getPosOfParagraph(paragraphToRemove));
}
}
private void setTextToRun(XWPFRun run, String stringsOnNewLines) {
ColoredText coloredText = getColor(stringsOnNewLines);
if (coloredText.getColor() != null) {
run.setTextHighlightColor(coloredText.getColor());
}
run.setText(coloredText.getText());
}
protected XWPFTable getRedactionTable(XWPFDocument doc) {
for (XWPFTable tbl : doc.getTables()) {

72
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java
View File

@ -10,9 +10,13 @@ import static com.iqser.red.service.redaction.report.v1.server.service.Placehold
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@ -25,6 +29,9 @@ import org.apache.commons.io.IOUtils;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mockito;
@ -70,7 +77,7 @@ import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@EnableAutoConfiguration(exclude = {/*StorageAutoConfiguration.class,*/RabbitAutoConfiguration.class})
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public class RedactionReportIntegrationTest {
@Autowired
@ -134,6 +141,7 @@ public class RedactionReportIntegrationTest {
return testDossier;
}
@Test
@SneakyThrows
public void testWordJustificationAppendixA1() {
@ -149,7 +157,6 @@ public class RedactionReportIntegrationTest {
var wordTemplateResource = new ClassPathResource("templates/Justification Appendix A1.docx");
var imageResource = new ClassPathResource("files/exampleImage.jpg");
var placeholders = buildPlaceHolderModel(Map.of("{{dossier.attribute.ActiveSubstance}}",
"Aktive Substanz \n Test Return",
"{{dossier.attribute.RapporteurMemberState}}",
@ -213,6 +220,48 @@ public class RedactionReportIntegrationTest {
@SneakyThrows
public void testWordIUCLIDFile() {
XWPFDocument doc = getIUCLIDFile();
List<XWPFParagraph> paragraphs = doc.getParagraphs();
Assertions.assertEquals(19, paragraphs.size());
List<String> expectedContent = getExpectedContent("expected/iuclid.txt");
List<String> contentOfParagraphs = getContentOfParagraphs(paragraphs);
assertThat(contentOfParagraphs).isEqualTo(expectedContent);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/IUCLID_Template_justification.docx")) {
fileOutputStream.write(wordReportGenerationService.toByteArray(doc));
}
}
private List<String> getExpectedContent(String path) throws IOException {
ClassPathResource classPathResource = new ClassPathResource(path);
InputStream inputStream = classPathResource.getInputStream();
List<String> expectedContent = IOUtils.readLines(inputStream, "UTF-8");
return expectedContent;
}
private List<String> getContentOfParagraphs(List<XWPFParagraph> paragraphs) {
List<String> res = new ArrayList<>();
for (XWPFParagraph paragraph : paragraphs) {
for (XWPFRun run : paragraph.getRuns()) {
res.add(run.text());
}
}
return res;
}
private XWPFDocument getIUCLIDFile() throws IOException {
Dossier dossier = prepareDossier();
FileModel fileStatus = FileModel.builder().filename("filename").build();
@ -227,11 +276,11 @@ public class RedactionReportIntegrationTest {
var placeholders = buildPlaceHolderModel(new HashMap<>(), Map.of("{{file.attribute.Path}}", "Path"), List.of());
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/IUCLID_Template_justification.docx")) {
fileOutputStream.write(report);
}
byte[] report = wordReportGenerationService.toByteArray(doc);
XWPFDocument newDoc = new XWPFDocument(new ByteArrayInputStream(report));
return newDoc;
}
@ -275,7 +324,10 @@ public class RedactionReportIntegrationTest {
FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build();
RedactionLog redactionLogSecondFile = objectMapper.readValue(new ClassPathResource("files/excelReportRedactionLog.json").getInputStream(), RedactionLog.class);
List<ReportRedactionEntry> reportEntriesSecondFile = redactionLogConverterService.convertAndSort(redactionLogSecondFile, legalBasisMapping, new HashMap<>(), dossier.getDossierId());
List<ReportRedactionEntry> reportEntriesSecondFile = redactionLogConverterService.convertAndSort(redactionLogSecondFile,
legalBasisMapping,
new HashMap<>(),
dossier.getDossierId());
ClassPathResource templateResource = new ClassPathResource("templates/Seeds-NewJustificationForm.docx");
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
@ -342,7 +394,6 @@ public class RedactionReportIntegrationTest {
Map<String, String> mapOfEntityDisplayName = createEntityDisplayNames(redactionLog);
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping, mapOfEntityDisplayName, dossier.getDossierId());
ClassPathResource templateResource = new ClassPathResource("templates/Excel Report.xlsx");
var placeholders = buildPlaceHolderModel(new HashMap<>(), new HashMap<>(), List.of());
@ -385,7 +436,10 @@ public class RedactionReportIntegrationTest {
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName", fileModel, excelModel, false);
RedactionLog redactionLogSecondFile = objectMapper.readValue(new ClassPathResource("files/excelReportRedactionLog.json").getInputStream(), RedactionLog.class);
List<ReportRedactionEntry> reportEntriesSecondFile = redactionLogConverterService.convertAndSort(redactionLogSecondFile, legalBasisMapping, mapOfEntityDisplayName, dossier.getDossierId());
List<ReportRedactionEntry> reportEntriesSecondFile = redactionLogConverterService.convertAndSort(redactionLogSecondFile,
legalBasisMapping,
mapOfEntityDisplayName,
dossier.getDossierId());
FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build();
excelTemplateReportGenerationService.generateExcelReport(reportEntriesSecondFile,
placeholders,

19
redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/expected/iuclid.txt Normal file
View File

File diff suppressed because one or more lines are too long