Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

RED-1186: updates

Browse Source
This commit is contained in:
aoezyetimoglu 2021-05-04 11:53:13 +02:00
parent d0484e1097
commit 5c3f69e005
5 changed files with 116 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/client/ProjectClient.java Normal file
View File

@ -0,0 +1,9 @@
package com.iqser.red.service.redaction.report.v1.server.client;
import com.iqser.red.service.file.management.v1.api.resources.ProjectResource;
import org.springframework.cloud.openfeign.FeignClient;
@FeignClient(name = "ProjectResource", url = "${file-management-service.url}")
public interface ProjectClient extends ProjectResource {
}

13
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java
View File

@ -2,11 +2,13 @@ package com.iqser.red.service.redaction.report.v1.server.service;
import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping;
import com.iqser.red.service.file.management.v1.api.model.FileStatus;
import com.iqser.red.service.file.management.v1.api.model.Project;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.service.redaction.report.v1.server.client.FileStatusClient;
import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient;
import com.iqser.red.service.redaction.report.v1.server.client.ProjectClient;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
@ -31,6 +33,7 @@ public class ReportGenerationService {
private final LegalBasisMappingClient legalBasisMappingClient;
private final RedactionLogConverterService redactionLogConverterService;
private final FileStatusClient fileStatusClient;
private final ProjectClient projectClient;
public List<StoredFileInformation> generateReport(ReportRequestMessage reportMessage) {
@ -47,6 +50,8 @@ public class ReportGenerationService {
List<StoredFileInformation> storedFileInformation = new ArrayList<>();
Project project = projectClient.getProjectById(reportMessage.getProjectId());
int i = 1;
for (String fileId : reportMessage.getFileIds()) {
@ -71,15 +76,15 @@ public class ReportGenerationService {
}
if (reportMessage.getReportTypes().contains(ReportType.WORD_SINGLE_FILE_APPENDIX_A1_TEMPLATE)) {
byte[] wordEFSATemplate = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_APPENDIX_A1_TEMPLATE, reportEntries, fileStatus
.getFilename());
byte[] wordEFSATemplate = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_APPENDIX_A1_TEMPLATE,
reportEntries, redactionLog.getRuleSetId(), fileStatus, project);
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordEFSATemplate);
storedFileInformation.add(new StoredFileInformation(fileId, storageId, ReportType.WORD_SINGLE_FILE_APPENDIX_A1_TEMPLATE));
}
if (reportMessage.getReportTypes().contains(ReportType.WORD_SINGLE_FILE_APPENDIX_A2_TEMPLATE)) {
byte[] wordSyngentaTemplate = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_APPENDIX_A2_TEMPLATE, reportEntries, fileStatus
.getFilename());
byte[] wordSyngentaTemplate = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_APPENDIX_A2_TEMPLATE,
reportEntries, redactionLog.getRuleSetId(), fileStatus, project);
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordSyngentaTemplate);
storedFileInformation.add(new StoredFileInformation(fileId, storageId, ReportType.WORD_SINGLE_FILE_APPENDIX_A2_TEMPLATE));
}

208
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java
View File

@ -1,29 +1,24 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.regex.Pattern;
import javax.annotation.PostConstruct;
import com.iqser.red.service.configuration.v1.api.model.FileAttributesConfig;
import com.iqser.red.service.redaction.report.v1.server.client.*;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.util.Units;
import com.iqser.red.service.file.management.v1.api.model.FileStatus;
import com.iqser.red.service.file.management.v1.api.model.Project;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesClient;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.utils.ResourceLoader;
import lombok.RequiredArgsConstructor;
import org.apache.poi.xwpf.usermodel.*;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.utils.ResourceLoader;
import lombok.RequiredArgsConstructor;
import javax.annotation.PostConstruct;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.regex.Pattern;
@Service
@RequiredArgsConstructor
@ -31,7 +26,7 @@ public class WordReportGenerationService {
private byte[] appendixA1Template;
private byte[] appendixA2Template;
private FileAttributesClient fileAttributesClient;
private final FileAttributesClient fileAttributesClient;
public static final String FILE_NAME_PLACEHOLDER = "{{file.name}}";
public static final String PAGE_PLACEHOLDER = "{{redaction.page}}";
@ -50,25 +45,19 @@ public class WordReportGenerationService {
public static final String DOSSIER_NAME_PLACEHOLDER = "{{dossier.name}}";
List<String> placeholders = List.of(FILE_NAME_PLACEHOLDER, FORMAT_DATE_ISO_PLACEHOLDER, FORMAT_DATE_GER_PLACEHOLDER, FORMAT_DATE_ENG_PLACEHOLDER, FORMAT_TIME_ISO_PLACEHOLDER, DOSSIER_NAME_PLACEHOLDER);
@PostConstruct
public void init() {
appendixA1Template = ResourceLoader.load("templates/Sanitisation justification Appendix A1.docx");
appendixA2Template = ResourceLoader.load("templates/Sanitisation justification Appendix A2.docx");
}
public byte[] generateReport(ReportType reportType, List<ReportRedactionEntry> reportEntries, String filename, String ruleSetId) {
Map<String, String> placeholders = new HashMap<>();
FileAttributesConfig fileAttributesConfig = fileAttributesClient.getFileAttributes(ruleSetId);
fileAttributesConfig.getFileAttributeConfigs().forEach(fileAttributeConfig -> {
String placeholder = "{{file.attribute" + fileAttributeConfig.getLabel().replace(" ", "-").toLowerCase(Locale.ROOT) + "}}";
placeholders.put(placeholder, fileAttributeConfig.getId());
});
public byte[] generateReport(ReportType reportType, List<ReportRedactionEntry> reportEntries, String ruleSetId, FileStatus fileStatus, Project project) {
List<String> placeholders = getDefaultPlaceholders();
Map<String, String> fileAttributePlaceholders = getFileAttributePlaceholders(ruleSetId);
placeholders.addAll(fileAttributePlaceholders.keySet());
byte[] template;
@ -82,23 +71,10 @@ public class WordReportGenerationService {
try (ByteArrayInputStream is = new ByteArrayInputStream(template)) {
XWPFDocument doc = new XWPFDocument(is);
addTableRows(doc, reportEntries, fileStatus.getFilename());
for(String placeholder: placeholders) {
replaceTextPlaceholders(doc, placeholder, getPlaceholderValue(placeholder, , , placeholders));
replaceTextPlaceholders(doc, placeholder, getPlaceholderValue(placeholder, project, fileStatus, fileAttributePlaceholders));
}
for (ImagePlaceholder imagePlaceholder: placeholders.getImagePlaceholders()){
replaceImagePlaceholders(doc, imagePlaceholder);
}
for (Placeholder placeholder : placeholders.getTextPlaceholders()) {
replaceTextPlaceholders(doc, placeholder.getPlaceholder(), placeholder.getValue());
}
for (DatePlaceholder datePlaceholder: placeholders.getDatePlaceholders()) {
replaceDatePlaceholder(doc, datePlaceholder.getPlaceholder());
}
for (TimePlaceholder timePlaceholder: placeholders.getTimePlaceholders()) {
replaceTimePlaceholder(doc, timePlaceholder.getPlaceholder());
}
addTableRows(doc, reportEntries, filename);
return toByteArray(doc);
} catch (IOException e) {
throw new RuntimeException(e);
@ -107,7 +83,24 @@ public class WordReportGenerationService {
}
}
private String getPlaceholderValue(String placeholder, String dossierName, String fileName, Map<String,String> placeholders) {
private List<String> getDefaultPlaceholders() {
List<String> defPlaceholders = new ArrayList<>();
defPlaceholders.addAll(Arrays.asList(FILE_NAME_PLACEHOLDER, FORMAT_DATE_ISO_PLACEHOLDER, FORMAT_DATE_GER_PLACEHOLDER, FORMAT_DATE_ENG_PLACEHOLDER, FORMAT_TIME_ISO_PLACEHOLDER, DOSSIER_NAME_PLACEHOLDER));
return defPlaceholders;
}
private Map<String, String> getFileAttributePlaceholders(String ruleSetId) {
Map<String, String> fileAttributePlaceholders = new HashMap<>();
FileAttributesConfig fileAttributesConfig = fileAttributesClient.getFileAttributes(ruleSetId);
fileAttributesConfig.getFileAttributeConfigs().forEach(fileAttributeConfig -> {
String fileAttributePlaceholderName = "{{file.attribute." + fileAttributeConfig.getLabel().replace(" ", "-").toLowerCase(Locale.ROOT) + "}}";
fileAttributePlaceholders.put(fileAttributePlaceholderName, fileAttributeConfig.getId());
});
return fileAttributePlaceholders;
}
private String getPlaceholderValue(String placeholder, Project project, FileStatus fileStatus, Map<String,String> fileAttributePlaceholders) {
if (placeholder.equals(FORMAT_DATE_ISO_PLACEHOLDER)) {
return OffsetDateTime.now().format(FORMAT_DATE_ISO);
}
@ -121,84 +114,19 @@ public class WordReportGenerationService {
return OffsetDateTime.now().format(FORMAT_TIME_ISO);
}
if (placeholder.equals(DOSSIER_NAME_PLACEHOLDER)) {
return dossierName;
return project.getProjectName();
}
if (placeholder.equals(FILE_NAME_PLACEHOLDER)) {
return fileName;
return fileStatus.getFilename();
}
return null;
if (fileAttributePlaceholders.containsKey(placeholder)) {
String id = fileAttributePlaceholders.get(placeholder);
return fileStatus.getFileAttributes().getAttributeIdToValue().get(id);
}
throw new RuntimeException("unknown placeholder");
}
private void replaceTimePlaceholder(XWPFDocument doc, String timePlaceholder) {
replaceParagraphForTimePlaceholder(doc.getParagraphs(), timePlaceholder);
for (XWPFTable tbl : doc.getTables()) {
for (XWPFTableRow row : tbl.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
replaceParagraphForTimePlaceholder(cell.getParagraphs(), timePlaceholder);
}
}
}
}
private void replaceParagraphForTimePlaceholder(List<XWPFParagraph> paragraphs, String timePlaceholder) {
for (XWPFParagraph p : paragraphs) {
String paragraphText = p.getText();
if (paragraphText.contains(timePlaceholder)) {
String safeToUseInReplaceAllString = Pattern.quote(timePlaceholder);
// String replace = timePlaceholder.substring(timePlaceholder.length()-7, timePlaceholder.length()-2);
String replace = OffsetDateTime.now().truncatedTo(ChronoUnit.MINUTES).toLocalTime().toString();
paragraphText = paragraphText.replaceAll(safeToUseInReplaceAllString, replace);
XWPFRun run = p.getRuns().get(0);
run.setText(paragraphText, 0);
int size = p.getRuns().size();
for (int i = 1; i < size; i++) {
p.removeRun(1);
}
}
}
}
private void replaceDatePlaceholder(XWPFDocument doc, String datePlaceholder) {
replaceParagraphForDatePlaceholder(doc.getParagraphs(), datePlaceholder);
for (XWPFTable tbl : doc.getTables()) {
for (XWPFTableRow row : tbl.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
replaceParagraphForDatePlaceholder(cell.getParagraphs(), datePlaceholder);
}
}
}
}
private void replaceParagraphForDatePlaceholder(List<XWPFParagraph> paragraphs, String datePlaceholder) {
for (XWPFParagraph p : paragraphs) {
String paragraphText = p.getText();
if (paragraphText.contains(datePlaceholder)) {
String safeToUseInReplaceAllString = Pattern.quote(datePlaceholder);
// String replace = datePlaceholder.substring(datePlaceholder.length()-12, datePlaceholder.length()-2);
String replace = OffsetDateTime.now().toLocalDate().toString();
paragraphText = paragraphText.replaceAll(safeToUseInReplaceAllString, replace);
XWPFRun run = p.getRuns().get(0);
run.setText(paragraphText, 0);
int size = p.getRuns().size();
for (int i = 1; i < size; i++) {
p.removeRun(1);
}
}
}
}
private void replaceImagePlaceholders(XWPFDocument doc, ImagePlaceholder imagePlaceholder) {
replaceParagraphForImagePlaceholder(doc.getParagraphs(), imagePlaceholder);
for (XWPFTable tbl: doc.getTables()) {
for (XWPFTableRow row: tbl.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
replaceParagraphForImagePlaceholder(cell.getParagraphs(), imagePlaceholder);
}
}
}
}
private void replaceParagraphForImagePlaceholder(List<XWPFParagraph> paragraphs, ImagePlaceholder imagePlaceholder) {
/*private void replaceParagraphForImagePlaceholder(List<XWPFParagraph> paragraphs, ImagePlaceholder imagePlaceholder) {
for (XWPFParagraph p : paragraphs) {
String paragraphText = p.getText();
if (paragraphText.contains(imagePlaceholder.getPlaceholder())) {
@ -212,20 +140,20 @@ public class WordReportGenerationService {
}
}
}
}
}*/
public void replaceTextPlaceholders(XWPFDocument doc, String search, String replace) {
replaceParagraphForTextPlaceholder(doc.getParagraphs(), search, replace);
replacePlaceholderInParagraph(doc.getParagraphs(), search, replace);
for (XWPFTable tbl : doc.getTables()) {
for (XWPFTableRow row : tbl.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
replaceParagraphForTextPlaceholder(cell.getParagraphs(), search, replace);
replacePlaceholderInParagraph(cell.getParagraphs(), search, replace);
}
}
}
}
private void replaceParagraphForTextPlaceholder(List<XWPFParagraph> paragraphs, String search, String replace) {
private void replacePlaceholderInParagraph(List<XWPFParagraph> paragraphs, String search, String replace) {
for (XWPFParagraph p : paragraphs) {
String paragraphText = p.getText();
if (paragraphText.contains(search)) {
@ -241,41 +169,45 @@ public class WordReportGenerationService {
}
}
private XWPFTable getTableWithTextPlaceholder(XWPFDocument doc) {
private XWPFTable getRedactionTable(XWPFDocument doc) {
for (XWPFTable tbl : doc.getTables()) {
String tblText = tbl.getText();
if (containsTextPlaceholder(tblText)) {
if (tblText.contains(PAGE_PLACEHOLDER)
|| tblText.contains(PARAGRAPH_PLACEHOLDER)
|| tblText.contains(JUSTIFICATION_PLACEHOLDER)) {
return tbl;
}
}
return null;
}
private boolean containsTextPlaceholder(String place) {
return place.contains(FILENAME_PLACEHOLDER)
|| place.contains(PAGE_PLACEHOLDER)
|| place.contains(PARAGRAPH_PLACEHOLDER)
|| place.contains(JUSTIFICATION_PLACEHOLDER);
private boolean containsRedactionPlaceholder(String text) {
return text.contains(FILE_NAME_PLACEHOLDER)
|| text.contains(PAGE_PLACEHOLDER)
|| text.contains(PARAGRAPH_PLACEHOLDER)
|| text.contains(JUSTIFICATION_PLACEHOLDER);
}
private void addTableRows(XWPFDocument doc, List<ReportRedactionEntry> reportEntries, String filename) {
List<String> finalPlaceholders = new ArrayList<>();
finalPlaceholders.addAll(Arrays.asList(FILENAME_PLACEHOLDER, PAGE_PLACEHOLDER, PARAGRAPH_PLACEHOLDER, JUSTIFICATION_PLACEHOLDER));
finalPlaceholders.addAll(Arrays.asList(FILE_NAME_PLACEHOLDER, PAGE_PLACEHOLDER, PARAGRAPH_PLACEHOLDER, JUSTIFICATION_PLACEHOLDER));
XWPFTable table = getRedactionTable(doc);
if (table == null) {
return;
}
XWPFTable table = getTableWithTextPlaceholder(doc);
Map<Integer,String> placeholderCellPos = new HashMap<>();
int placeholderRow = -1;
for (int j=0; j<table.getRows().size(); j++) {
for (int i=0; i<table.getRows().get(j).getTableCells().size(); i++) {
XWPFTableCell cell = table.getRows().get(j).getTableCells().get(i);
finalPlaceholders.forEach(ph -> replaceParagraphForTextPlaceholder(cell.getParagraphs(), ph, ph));
if (containsTextPlaceholder(cell.getText())) {
if (containsRedactionPlaceholder(cell.getText())) {
placeholderCellPos.put(i, cell.getText());
placeholderRow = j;
}
}
}
table.removeRow(placeholderRow);
@ -288,7 +220,7 @@ public class WordReportGenerationService {
}
private String replaceTextPlaceholderWithEntries(ReportRedactionEntry entry, String filename, String placeholder) {
if(placeholder.equals(FILENAME_PLACEHOLDER)) {
if(placeholder.equals(FILE_NAME_PLACEHOLDER)) {
return filename;
}
if(placeholder.equals(PAGE_PLACEHOLDER)) {

BIN
redaction-report-service-v1/redaction-report-service-server-v1/src/main/resources/templates/Sanitisation justification Appendix A1.docx
View File

Binary file not shown.

30
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java
View File

@ -1,11 +1,21 @@
package com.iqser.red.service.redaction.report.v1.server;
import static org.mockito.Mockito.when;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.iqser.red.service.configuration.v1.api.model.FileAttributeConfig;
import com.iqser.red.service.configuration.v1.api.model.FileAttributesConfig;
import com.iqser.red.service.file.management.v1.api.model.FileAttributes;
import com.iqser.red.service.file.management.v1.api.model.FileStatus;
import com.iqser.red.service.file.management.v1.api.model.Project;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesClient;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@ -41,6 +51,9 @@ public class RedactionReportIntegrationTest {
@MockBean
private StorageService storageService;
@MockBean
private FileAttributesClient fileAttributesClient;
@MockBean
private AmazonS3 s3Client;
@ -73,8 +86,21 @@ public class RedactionReportIntegrationTest {
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping);
byte[] report = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_APPENDIX_A2_TEMPLATE, reportEntries, redactionLog
.getFilename());
List<FileAttributeConfig> fileAttributeConfigs = new ArrayList<>();
fileAttributeConfigs.add(new FileAttributeConfig("3e9b9569-5d2e-4619-848b-dd0a3e96527f", "", "Document Title", true, true));
FileAttributesConfig fileAttributesConfig = new FileAttributesConfig("", "", fileAttributeConfigs);
when(fileAttributesClient.getFileAttributes(redactionLog.getRuleSetId())).thenReturn(fileAttributesConfig);
Map<String, String> attributeIdToValue = new HashMap();
attributeIdToValue.put("3e9b9569-5d2e-4619-848b-dd0a3e96527f", "Test");
FileAttributes fileAttributes = new FileAttributes(attributeIdToValue);
FileStatus fileStatus = FileStatus.builder().filename("filename").fileAttributes(fileAttributes).build();
Project project = Project.builder().projectId("projectId").projectName("projectName").build();
byte[] report = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_APPENDIX_A1_TEMPLATE, reportEntries, redactionLog
.getRuleSetId(), fileStatus, project);
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/efsa_template.docx")) {
fileOutputStream.write(report);