Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

Pull request #16: RED-1058: Added generation of excel report, communitate via queue, load redaction log files direct from storage, sort report entries the same way as the ui does

Browse Source 
Merge in RED/redaction-report-service from RED-1058 to master

* commit 'e29a5b91f5a3c24bd0b6b0e4198fe4403d9cea7e':
  RED-1058: Added generation of excel report, communitate via queue, load redaction log files direct from storage, sort report entries the same way as the ui does
This commit is contained in:
Dominique Eiflaender 2021-02-22 12:42:15 +01:00
commit 24195c74cc
21 changed files with 692 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-report-service-v1/pom.xml
View File

@ -26,7 +26,7 @@
<dependency>
<groupId>com.iqser.red</groupId>
<artifactId>platform-commons-dependency</artifactId>
<version>1.1.1</version>
<version>1.2.2</version>
<scope>import</scope>
<type>pom</type>
</dependency>

33
redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportRequestMessage.java Normal file
View File

@ -0,0 +1,33 @@
package com.iqser.red.service.redaction.report.v1.api.model;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ReportRequestMessage {
private String userId;
private String ruleSetId;
private String downloadId;
private String projectId;
@Builder.Default
private List<String> fileIds = new ArrayList<>();
@Builder.Default
private Set<ReportType> reportTypes = new HashSet<>();
}

13
redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportResult.java
View File

@ -1,13 +0,0 @@
package com.iqser.red.service.redaction.report.v1.api.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class ReportResult {
private byte[] document;
}

14
redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/MultiFileRedactionLog.java → redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportResultMessage.java
View File

@ -3,21 +3,19 @@ package com.iqser.red.service.redaction.report.v1.api.model;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class MultiFileRedactionLog {
public class ReportResultMessage {
private List<RedactionLog> redactionLogs = new ArrayList<>();
private String template;
private String ruleSetId;
private String userId;
private String downloadId;
private List<StoredFileInformation> storedFileInformation = new ArrayList<>();
}

5
redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportType.java Normal file
View File

@ -0,0 +1,5 @@
package com.iqser.red.service.redaction.report.v1.api.model;
public enum ReportType {
WORD_SINGLE_FILE_EFSA_TEMPLATE, WORD_SINGLE_FILE_SYNGENTA_TEMPLATE, EXCEL_MULTI_FILE, EXCEL_SINGLE_FILE
}

19
redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java Normal file
View File

@ -0,0 +1,19 @@
package com.iqser.red.service.redaction.report.v1.api.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class StoredFileInformation {
private String fileId;
private String originalFilename;
private String storageId;
private ReportType reportType;
}

17
redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/resource/RedactionReportResource.java
View File

@ -1,17 +0,0 @@
package com.iqser.red.service.redaction.report.v1.api.resource;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.redaction.report.v1.api.model.MultiFileRedactionLog;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResult;
public interface RedactionReportResource {
String SERVICE_NAME = "redaction-report-service-v1";
@PostMapping(value = "/generate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
ReportResult generateReport(@RequestBody MultiFileRedactionLog multiFileRedactionLog);
}

19
redaction-report-service-v1/redaction-report-service-server-v1/pom.xml
View File

@ -39,6 +39,18 @@
<artifactId>poi-scratchpad</artifactId>
<version>4.1.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
<version>2.3.1.RELEASE</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>storage-commons</artifactId>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>spring-commons</artifactId>
@ -58,6 +70,13 @@
<artifactId>test-commons</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.amqp</groupId>
<artifactId>spring-rabbit-test</artifactId>
<version>2.3.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>

7
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java
View File

@ -5,19 +5,17 @@ import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvc
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Import;
import org.springframework.scheduling.annotation.EnableAsync;
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient;
import com.iqser.red.service.redaction.report.v1.server.settings.RedactionReportSettings;
import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration;
@EnableAsync
@EnableConfigurationProperties(RedactionReportSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
@Import(DefaultWebMvcConfiguration.class)
@Import({DefaultWebMvcConfiguration.class, MessagingConfiguration.class})
@EnableFeignClients(basePackageClasses = {LegalBasisMappingClient.class})
public class Application {
@ -27,6 +25,7 @@ public class Application {
* @param args Any command line parameter given upon startup.
*/
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}

56
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/configuration/MessagingConfiguration.java Normal file
View File

@ -0,0 +1,56 @@
package com.iqser.red.service.redaction.report.v1.server.configuration;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import lombok.RequiredArgsConstructor;
@Configuration
@RequiredArgsConstructor
public class MessagingConfiguration {
public static final String REPORT_QUEUE = "reportQueue";
public static final String REPORT_DLQ = "reportDLQ";
public static final String REPORT_RESULT_QUEUE = "reportResultQueue";
public static final String REPORT_RESULT_DLQ = "reportResultDLQ";
@Bean
public Queue reportQueue() {
return QueueBuilder.durable(REPORT_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", REPORT_DLQ)
.withArgument("x-max-priority", 2)
.maxPriority(2)
.build();
}
@Bean
public Queue reportDeadLetterQueue() {
return QueueBuilder.durable(REPORT_DLQ).build();
}
@Bean
public Queue reportResultQueue() {
return QueueBuilder.durable(REPORT_RESULT_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ)
.build();
}
@Bean
public Queue reportResultDeadLetterQueue() {
return QueueBuilder.durable(REPORT_RESULT_DLQ).build();
}
}

23
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/controller/RedactionReportController.java
View File

@ -1,23 +0,0 @@
package com.iqser.red.service.redaction.report.v1.server.controller;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.redaction.report.v1.api.model.MultiFileRedactionLog;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResult;
import com.iqser.red.service.redaction.report.v1.api.resource.RedactionReportResource;
import com.iqser.red.service.redaction.report.v1.server.service.ReportGenerationService;
import lombok.RequiredArgsConstructor;
@RestController
@RequiredArgsConstructor
public class RedactionReportController implements RedactionReportResource {
private final ReportGenerationService reportGenerationService;
public ReportResult generateReport(@RequestBody MultiFileRedactionLog multiFileRedactionLog){
return new ReportResult(reportGenerationService.generateReport(multiFileRedactionLog));
}
}

15
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ReportRedactionEntry.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class ReportRedactionEntry {
private int page;
private float x;
private float y;
private String section;
private String justification;
}

98
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java Normal file
View File

@ -0,0 +1,98 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.io.ByteArrayOutputStream;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@Service
@RequiredArgsConstructor
public class ExcelReportGenerationService {
public XSSFWorkbook createWorkbook() {
XSSFWorkbook workbook = new XSSFWorkbook();
return workbook;
}
public XSSFSheet createExcelSheet(XSSFWorkbook workbook) {
XSSFSheet sheet = workbook.createSheet("Justifications");
addHeader(sheet);
return sheet;
}
public byte[] generateSingleFileReport(List<ReportRedactionEntry> reportEntries, String filename) {
XSSFWorkbook workbook = new XSSFWorkbook();
XSSFSheet sheet = workbook.createSheet("Justifications");
addHeader(sheet);
AtomicInteger rowIndex = new AtomicInteger(1);
addEntries(sheet, reportEntries, filename, rowIndex);
return toByteArray(workbook);
}
public void addEntries(XSSFSheet sheet, List<ReportRedactionEntry> reportEntries, String filename,
AtomicInteger rowIndex) {
reportEntries.forEach(entry -> {
XSSFRow row = sheet.createRow(rowIndex.get());
XSSFCell filenameCell = row.createCell(0);
filenameCell.setCellValue(filename);
XSSFCell pageColumn = row.createCell(1);
pageColumn.setCellValue(entry.getPage());
XSSFCell paragraphColumn = row.createCell(2);
paragraphColumn.setCellValue(entry.getSection());
XSSFCell justificationColumn = row.createCell(3);
justificationColumn.setCellValue(entry.getJustification());
rowIndex.getAndIncrement();
});
}
private void addHeader(XSSFSheet sheet) {
sheet.setColumnWidth(0, 10000);
sheet.setColumnWidth(1, 4000);
sheet.setColumnWidth(2, 20000);
sheet.setColumnWidth(3, 40000);
XSSFRow row = sheet.createRow(0);
XSSFCell filename = row.createCell(0);
filename.setCellValue("Volume or Document Name");
XSSFCell page = row.createCell(1);
page.setCellValue("Electronic page No.");
XSSFCell paragraph = row.createCell(2);
paragraph.setCellValue("Paragraph");
XSSFCell justification = row.createCell(3);
justification.setCellValue("Justification and reference to Article 63 of Regulation (EC) No 1107/2009");
}
@SneakyThrows
public byte[] toByteArray(XSSFWorkbook doc) {
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
doc.write(byteArrayOutputStream);
doc.close();
return byteArrayOutputStream.toByteArray();
}
}
}

67
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/RedactionLogConverterService.java Normal file
View File

@ -0,0 +1,67 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.util.ArrayList;
import java.util.List;
import org.springframework.stereotype.Service;
import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.Status;
@Service
public class RedactionLogConverterService {
public List<ReportRedactionEntry> convertAndSort(RedactionLog redactionLog,
List<LegalBasisMapping> legalBasisMappings) {
List<ReportRedactionEntry> reportEntries = new ArrayList<>();
redactionLog.getRedactionLogEntry().forEach(entry -> {
if (entry.isRedacted()) {
if (entry.isRecommendation()) {
return;
}
if (entry.isManual() && entry.getManualRedactionType()
.equals(ManualRedactionType.ADD) && !entry.getStatus().equals(Status.APPROVED)) {
return;
}
if (entry.isManual() && entry.getManualRedactionType()
.equals(ManualRedactionType.REMOVE) && entry.getStatus().equals(Status.APPROVED)) {
return;
}
entry.getPositions().forEach(position -> {
reportEntries.add(new ReportRedactionEntry(position.getPage(), position.getTopLeft()
.getX(), position.getTopLeft()
.getY(), entry.getSection(), entry.getLegalBasis() + " " + legalBasisMappings.stream()
.filter(lbm -> lbm.getReason().equalsIgnoreCase(entry.getLegalBasis()))
.findAny()
.map(LegalBasisMapping::getDescription)
.orElse("")));
});
}
});
reportEntries.sort((entry1, entry2) -> {
if (entry1.getPage() == entry2.getPage()) {
if (entry1.getY() == entry2.getY()) {
return entry1.getX() < entry2.getX() ? 1 : -1;
} else {
return entry1.getY() < entry2.getY() ? 1 : -1;
}
}
return entry1.getPage() < entry2.getPage() ? -1 : 1;
});
return reportEntries;
}
}

199
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java
View File

@ -1,146 +1,99 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping;
import com.iqser.red.service.redaction.report.v1.api.model.MultiFileRedactionLog;
import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient;
import com.iqser.red.service.redaction.report.v1.server.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.Status;
import lombok.RequiredArgsConstructor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;
import org.apache.xmlbeans.XmlCursor;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class ReportGenerationService {
private final ReportStorageService reportStorageService;
private final ExcelReportGenerationService excelReportService;
private final WordReportGenerationService wordReportGenerationService;
private final LegalBasisMappingClient legalBasisMappingClient;
private byte[] efsaTemplate;
private byte[] syngentaTemplate;
private final RedactionLogConverterService redactionLogConverterService;
@PostConstruct
public void init() {
public List<StoredFileInformation> generateReport(ReportRequestMessage reportMessage) {
efsaTemplate = ResourceLoader.load("templates/EFSA Template.docx");
syngentaTemplate = ResourceLoader.load("templates/Syngenta Template.docx");
}
List<LegalBasisMapping> legalBasisMappings = legalBasisMappingClient.getLegalBasisMapping(reportMessage.getRuleSetId());
XSSFWorkbook excelMultiFileWorkbook = null;
XSSFSheet excelMultiFileSheet = null;
AtomicInteger excelRowIndex = new AtomicInteger(1);
public byte[] generateReport(MultiFileRedactionLog multiFileRedactionLog) {
byte[] template;
if (multiFileRedactionLog.getTemplate().equals("EFSA Template")) {
template = efsaTemplate;
} else if (multiFileRedactionLog.getTemplate().equals("Syngenta Template")) {
template = syngentaTemplate;
} else {
template = syngentaTemplate;
if (reportMessage.getReportTypes().contains(ReportType.EXCEL_MULTI_FILE)) {
excelMultiFileWorkbook = excelReportService.createWorkbook();
excelMultiFileSheet = excelReportService.createExcelSheet(excelMultiFileWorkbook);
}
try (ByteArrayInputStream is = new ByteArrayInputStream(template)) {
XWPFDocument doc = new XWPFDocument(is);
List<LegalBasisMapping> legalBasisMappings = legalBasisMappingClient.getLegalBasisMapping(multiFileRedactionLog.getRuleSetId());
addTableRows(doc, multiFileRedactionLog, legalBasisMappings);
return toByteArray(doc);
} catch (IOException e) {
throw new RuntimeException(e);
List<StoredFileInformation> storedFileInformation = new ArrayList<>();
int i = 1;
for (String fileId : reportMessage.getFileIds()) {
long start = System.currentTimeMillis();
RedactionLog redactionLog = reportStorageService.getRedactionLog(reportMessage.getProjectId(), fileId);
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMappings);
if (reportMessage.getReportTypes().contains(ReportType.EXCEL_MULTI_FILE)) {
excelReportService.addEntries(excelMultiFileSheet, reportEntries, redactionLog.getFilename(), excelRowIndex);
}
if (reportMessage.getReportTypes().contains(ReportType.EXCEL_SINGLE_FILE)) {
byte[] excelSingleReport = excelReportService.generateSingleFileReport(reportEntries, redactionLog.getFilename());
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), excelSingleReport);
storedFileInformation.add(new StoredFileInformation(fileId, redactionLog.getFilename(), storageId, ReportType.EXCEL_SINGLE_FILE));
}
if (reportMessage.getReportTypes().contains(ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE)) {
byte[] wordEFSATemplate = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE, reportEntries, redactionLog
.getFilename());
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordEFSATemplate);
storedFileInformation.add(new StoredFileInformation(fileId, redactionLog.getFilename(), storageId, ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE));
}
if (reportMessage.getReportTypes().contains(ReportType.WORD_SINGLE_FILE_SYNGENTA_TEMPLATE)) {
byte[] wordSyngentaTemplate = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_SYNGENTA_TEMPLATE, reportEntries, redactionLog
.getFilename());
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordSyngentaTemplate);
storedFileInformation.add(new StoredFileInformation(fileId, redactionLog.getFilename(), storageId, ReportType.WORD_SINGLE_FILE_SYNGENTA_TEMPLATE));
}
long end = System.currentTimeMillis();
log.info("Successfully processed {}/{} fileIds for downloadId {}, took {}", i, reportMessage.getFileIds()
.size(), reportMessage.getDownloadId(), end - start);
i++;
}
}
private void addTableRows(XWPFDocument doc, MultiFileRedactionLog multiFileRedactionLog,
List<LegalBasisMapping> legalBasisMappings) {
XWPFTable table = doc.getTables().get(0);
XmlCursor cursor = table.getCTTbl().newCursor();
XWPFParagraph newParagraph = doc.insertNewParagraph(cursor);
XWPFRun run = newParagraph.createRun();
run.setText("Applied rules: EFSA 1 (Vertebrate Authors)");
run.setFontSize(10);
multiFileRedactionLog.getRedactionLogs().stream().sorted(Comparator.comparing(RedactionLog::getFilename)).forEach(fileRedactionLog -> {
fileRedactionLog.getRedactionLogEntry().forEach(entry -> {
if (entry.isRedacted()) {
if (entry.isRecommendation()) {
return;
}
if (entry.isManual() && entry.getManualRedactionType()
.equals(ManualRedactionType.ADD) && !entry.getStatus().equals(Status.APPROVED)) {
return;
}
if (entry.isManual() && entry.getManualRedactionType()
.equals(ManualRedactionType.REMOVE) && entry.getStatus().equals(Status.APPROVED)) {
return;
}
List<Integer> pages = entry.getPositions()
.stream()
.map(Rectangle::getPage)
.distinct()
.sorted()
.collect(Collectors.toList());
pages.forEach(page -> {
XWPFTableRow row = table.createRow();
setText(row.getCell(0), fileRedactionLog.getFilename());
setText(row.getCell(1), String.valueOf(page));
setText(row.getCell(2), entry.getSection());
setText(row.getCell(3), entry.getLegalBasis() + " " +
legalBasisMappings
.stream()
.filter(lbm -> lbm.getReason().equalsIgnoreCase(entry.getLegalBasis()))
.findAny()
.map(LegalBasisMapping::getDescription)
.orElse(""));
});
}
});
});
}
private void setText(XWPFTableCell cell, String value) {
cell.setVerticalAlignment(XWPFTableCell.XWPFVertAlign.CENTER);
cell.removeParagraph(0);
XWPFParagraph addParagraph = cell.addParagraph();
XWPFRun run = addParagraph.createRun();
run.setFontSize(10);
run.setText(value);
}
private byte[] toByteArray(XWPFDocument doc) throws IOException {
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
doc.write(byteArrayOutputStream);
doc.close();
return byteArrayOutputStream.toByteArray();
if (reportMessage.getReportTypes().contains(ReportType.EXCEL_MULTI_FILE)) {
long start = System.currentTimeMillis();
byte[] multifileExcelReport = excelReportService.toByteArray(excelMultiFileWorkbook);
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), multifileExcelReport);
storedFileInformation.add(new StoredFileInformation(null, null, storageId, ReportType.EXCEL_MULTI_FILE));
long end = System.currentTimeMillis();
log.info("Successfully stored multiFileExcelReport for downloadId {}, took {}", reportMessage.getDownloadId(), end - start);
}
return storedFileInformation;
}
}

62
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportMessageReceiver.java Normal file
View File

@ -0,0 +1,62 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import static com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration.REPORT_QUEUE;
import static com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration.REPORT_RESULT_QUEUE;
import java.util.List;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@RabbitListener(queues = REPORT_QUEUE)
public class ReportMessageReceiver {
private final ObjectMapper objectMapper;
private final ReportGenerationService reportGenerationService;
private final RabbitTemplate rabbitTemplate;
@RabbitHandler
public void receive(String in) throws JsonProcessingException {
ReportRequestMessage reportMessage = objectMapper.readValue(in, ReportRequestMessage.class);
long start = System.currentTimeMillis();
log.info("Start generating reports for downloadId {}", reportMessage.getDownloadId());
List<StoredFileInformation> storedFileInformation = reportGenerationService.generateReport(reportMessage);
addToReportResultQueue(reportMessage.getUserId(), reportMessage.getDownloadId(), storedFileInformation, 1);
long end = System.currentTimeMillis();
log.info("Successfully generated reports for downloadId {}, took {}", reportMessage.getDownloadId(), end - start);
}
private void addToReportResultQueue(String userId, String downloadId,
List<StoredFileInformation> storedFileInformation, int priority) {
try {
rabbitTemplate.convertAndSend(REPORT_RESULT_QUEUE, objectMapper.writeValueAsString(new ReportResultMessage(userId, downloadId, storedFileInformation)), message -> {
message.getMessageProperties().setPriority(priority);
return message;
});
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
}

104
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java Normal file
View File

@ -0,0 +1,104 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import javax.annotation.PostConstruct;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;
import org.apache.xmlbeans.XmlCursor;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.utils.ResourceLoader;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class WordReportGenerationService {
private byte[] efsaTemplate;
private byte[] syngentaTemplate;
@PostConstruct
public void init() {
efsaTemplate = ResourceLoader.load("templates/EFSA Template.docx");
syngentaTemplate = ResourceLoader.load("templates/Syngenta Template.docx");
}
public byte[] generateReport(ReportType reportType, List<ReportRedactionEntry> reportEntries, String filename) {
byte[] template;
if (reportType.equals(ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE)) {
template = efsaTemplate;
} else if (reportType.equals(ReportType.WORD_SINGLE_FILE_SYNGENTA_TEMPLATE)) {
template = syngentaTemplate;
} else {
template = syngentaTemplate;
}
try (ByteArrayInputStream is = new ByteArrayInputStream(template)) {
XWPFDocument doc = new XWPFDocument(is);
addTableRows(doc, reportEntries, filename);
return toByteArray(doc);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private void addTableRows(XWPFDocument doc, List<ReportRedactionEntry> reportEntries, String filename) {
XWPFTable table = doc.getTables().get(0);
XmlCursor cursor = table.getCTTbl().newCursor();
XWPFParagraph newParagraph = doc.insertNewParagraph(cursor);
XWPFRun run = newParagraph.createRun();
run.setText("Applied rules: EFSA 1 (Vertebrate Authors)");
run.setFontSize(10);
reportEntries.forEach(entry -> {
XWPFTableRow row = table.createRow();
setText(row.getCell(0), filename);
setText(row.getCell(1), String.valueOf(entry.getPage()));
setText(row.getCell(2), entry.getSection());
setText(row.getCell(3), entry.getJustification());
});
}
private void setText(XWPFTableCell cell, String value) {
cell.setVerticalAlignment(XWPFTableCell.XWPFVertAlign.CENTER);
cell.removeParagraph(0);
XWPFParagraph addParagraph = cell.addParagraph();
XWPFRun run = addParagraph.createRun();
run.setFontSize(10);
run.setText(value);
}
private byte[] toByteArray(XWPFDocument doc) throws IOException {
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
doc.write(byteArrayOutputStream);
doc.close();
return byteArrayOutputStream.toByteArray();
}
}
}

11
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/RedactionReportSettings.java
View File

@ -1,11 +0,0 @@
package com.iqser.red.service.redaction.report.v1.server.settings;
import org.springframework.boot.context.properties.ConfigurationProperties;
import lombok.Data;
@Data
@ConfigurationProperties("service-template")
public class RedactionReportSettings {
private int numberOfParallelProcessed = 5;
}

40
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/ReportStorageService.java Normal file
View File

@ -0,0 +1,40 @@
package com.iqser.red.service.redaction.report.v1.server.storage;
import java.io.IOException;
import java.util.UUID;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class ReportStorageService {
private final StorageService storageService;
private final ObjectMapper objectMapper;
public RedactionLog getRedactionLog(String projectId, String fileId) {
var redactionLog = storageService.getObject(StorageIdUtils.getRedactionLogStorageId(projectId, fileId));
try {
return objectMapper.readValue(redactionLog.getInputStream(), RedactionLog.class);
} catch (IOException e) {
throw new RuntimeException("Could not convert RedactionLog", e);
}
}
public String storeObject(String downloadId, byte[] data) {
String storageId = StorageIdUtils.getStorageId(downloadId, UUID.randomUUID().toString());
storageService.storeObject(storageId, data);
return storageId;
}
}

16
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/StorageIdUtils.java Normal file
View File

@ -0,0 +1,16 @@
package com.iqser.red.service.redaction.report.v1.server.storage;
public class StorageIdUtils {
public static String getRedactionLogStorageId(String projectId, String fileId) {
return projectId + "/" + fileId + ".REDACTION_LOG.json";
}
public static String getStorageId(String downloadId, String tmpFilename) {
return downloadId + "/" + tmpFilename;
}
}

101
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java
View File

@ -1,13 +1,11 @@
package com.iqser.red.service.redaction.report.v1.server;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping;
import com.iqser.red.service.redaction.report.v1.api.model.MultiFileRedactionLog;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResult;
import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient;
import com.iqser.red.service.redaction.report.v1.server.controller.RedactionReportController;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@ -16,12 +14,19 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import static org.mockito.Mockito.when;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient;
import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.service.ExcelReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.service.RedactionLogConverterService;
import com.iqser.red.service.redaction.report.v1.server.service.WordReportGenerationService;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.service.StorageService;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = RANDOM_PORT)
@ -30,36 +35,74 @@ public class RedactionReportIntegrationTest {
@Autowired
private ObjectMapper objectMapper;
@Autowired
private RedactionReportController redactionReportController;
@MockBean
private LegalBasisMappingClient legalBasisMappingClient;
@MockBean
private StorageService storageService;
@MockBean
private AmazonS3 s3Client;
@Autowired
private WordReportGenerationService wordReportGenerationService;
@Autowired
private ExcelReportGenerationService excelReportGenerationService;
@MockBean
private MessagingConfiguration messagingConfiguration;
@Autowired
private RedactionLogConverterService redactionLogConverterService;
@Test
public void testReportGeneration() throws IOException {
ClassPathResource legalBasisMappingResource = new ClassPathResource("files/LegalBasisMapping.json");
List<LegalBasisMapping> legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() {
});
when(legalBasisMappingClient.getLegalBasisMapping("123")).thenReturn(legalBasisMapping);
public void testWordReportGeneration() throws IOException {
ClassPathResource redactionLogResource = new ClassPathResource("files/RedactedLog.txt");
RedactionLog redactionLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class);
redactionLog.setFilename("TestFile");
MultiFileRedactionLog multiFileRedactionLog = new MultiFileRedactionLog(List.of(redactionLog), "EFSA Template", "123");
ReportResult reportResult = redactionReportController.generateReport(multiFileRedactionLog);
ClassPathResource legalBasisMappingResource = new ClassPathResource("files/LegalBasisMapping.json");
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/document2.docx")) {
fileOutputStream.write(reportResult.getDocument());
List<LegalBasisMapping> legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping);
byte[] report = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE, reportEntries, redactionLog
.getFilename());
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/efsa_template.docx")) {
fileOutputStream.write(report);
}
}
@Test
public void testExcelReportGeneration() throws IOException {
ClassPathResource redactionLogResource = new ClassPathResource("files/RedactedLog.txt");
RedactionLog redactionLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class);
redactionLog.setFilename("TestFile");
ClassPathResource legalBasisMappingResource = new ClassPathResource("files/LegalBasisMapping.json");
List<LegalBasisMapping> legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping);
byte[] report = excelReportGenerationService.generateSingleFileReport(reportEntries, redactionLog.getFilename());
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/report.xlsx")) {
fileOutputStream.write(report);
}
}
}