diff --git a/redaction-report-service-v1/pom.xml b/redaction-report-service-v1/pom.xml index 1a020cd..229d25f 100644 --- a/redaction-report-service-v1/pom.xml +++ b/redaction-report-service-v1/pom.xml @@ -26,7 +26,7 @@ com.iqser.red platform-commons-dependency - 1.1.1 + 1.2.2 import pom diff --git a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportRequestMessage.java b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportRequestMessage.java new file mode 100644 index 0000000..ac4c3da --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportRequestMessage.java @@ -0,0 +1,33 @@ +package com.iqser.red.service.redaction.report.v1.api.model; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ReportRequestMessage { + + private String userId; + + private String ruleSetId; + + private String downloadId; + + private String projectId; + + @Builder.Default + private List fileIds = new ArrayList<>(); + + @Builder.Default + private Set reportTypes = new HashSet<>(); + +} diff --git a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportResult.java b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportResult.java deleted file mode 100644 index d10d383..0000000 --- a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportResult.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.iqser.red.service.redaction.report.v1.api.model; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ReportResult { - - private byte[] document; -} diff --git a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/MultiFileRedactionLog.java b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportResultMessage.java similarity index 52% rename from redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/MultiFileRedactionLog.java rename to redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportResultMessage.java index fe7865b..337e669 100644 --- a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/MultiFileRedactionLog.java +++ b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportResultMessage.java @@ -3,21 +3,19 @@ package com.iqser.red.service.redaction.report.v1.api.model; import java.util.ArrayList; import java.util.List; -import com.iqser.red.service.redaction.v1.model.RedactionLog; - import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; @Data +@Builder @AllArgsConstructor @NoArgsConstructor -public class MultiFileRedactionLog { +public class ReportResultMessage { - private List redactionLogs = new ArrayList<>(); - - private String template; - - private String ruleSetId; + private String userId; + private String downloadId; + private List storedFileInformation = new ArrayList<>(); } diff --git a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportType.java b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportType.java new file mode 100644 index 0000000..de5bf7d --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportType.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.redaction.report.v1.api.model; + +public enum ReportType { + WORD_SINGLE_FILE_EFSA_TEMPLATE, WORD_SINGLE_FILE_SYNGENTA_TEMPLATE, EXCEL_MULTI_FILE, EXCEL_SINGLE_FILE +} diff --git a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java new file mode 100644 index 0000000..a35f247 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.report.v1.api.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class StoredFileInformation { + + private String fileId; + private String originalFilename; + private String storageId; + private ReportType reportType; + +} diff --git a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/resource/RedactionReportResource.java b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/resource/RedactionReportResource.java deleted file mode 100644 index 89d20f3..0000000 --- a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/resource/RedactionReportResource.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.iqser.red.service.redaction.report.v1.api.resource; - -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; - -import com.iqser.red.service.redaction.report.v1.api.model.MultiFileRedactionLog; -import com.iqser.red.service.redaction.report.v1.api.model.ReportResult; - -public interface RedactionReportResource { - - String SERVICE_NAME = "redaction-report-service-v1"; - - @PostMapping(value = "/generate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) - ReportResult generateReport(@RequestBody MultiFileRedactionLog multiFileRedactionLog); - -} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/pom.xml b/redaction-report-service-v1/redaction-report-service-server-v1/pom.xml index 5b8a9bf..cc5e2ae 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/pom.xml +++ b/redaction-report-service-v1/redaction-report-service-server-v1/pom.xml @@ -39,6 +39,18 @@ poi-scratchpad 4.1.2 + + + org.springframework.boot + spring-boot-starter-amqp + 2.3.1.RELEASE + + + + + com.iqser.red.commons + storage-commons + com.iqser.red.commons spring-commons @@ -58,6 +70,13 @@ test-commons test + + + org.springframework.amqp + spring-rabbit-test + 2.3.1 + test + diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java index dabcf7b..f0ae299 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java @@ -5,19 +5,17 @@ import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvc import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.annotation.Import; import org.springframework.scheduling.annotation.EnableAsync; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient; -import com.iqser.red.service.redaction.report.v1.server.settings.RedactionReportSettings; +import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration; @EnableAsync -@EnableConfigurationProperties(RedactionReportSettings.class) @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class}) -@Import(DefaultWebMvcConfiguration.class) +@Import({DefaultWebMvcConfiguration.class, MessagingConfiguration.class}) @EnableFeignClients(basePackageClasses = {LegalBasisMappingClient.class}) public class Application { @@ -27,6 +25,7 @@ public class Application { * @param args Any command line parameter given upon startup. */ public static void main(String[] args) { + SpringApplication.run(Application.class, args); } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/configuration/MessagingConfiguration.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/configuration/MessagingConfiguration.java new file mode 100644 index 0000000..47ec712 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/configuration/MessagingConfiguration.java @@ -0,0 +1,56 @@ +package com.iqser.red.service.redaction.report.v1.server.configuration; + +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.core.QueueBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import lombok.RequiredArgsConstructor; + +@Configuration +@RequiredArgsConstructor +public class MessagingConfiguration { + + public static final String REPORT_QUEUE = "reportQueue"; + public static final String REPORT_DLQ = "reportDLQ"; + + public static final String REPORT_RESULT_QUEUE = "reportResultQueue"; + public static final String REPORT_RESULT_DLQ = "reportResultDLQ"; + + + @Bean + public Queue reportQueue() { + + return QueueBuilder.durable(REPORT_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", REPORT_DLQ) + .withArgument("x-max-priority", 2) + .maxPriority(2) + .build(); + } + + + @Bean + public Queue reportDeadLetterQueue() { + + return QueueBuilder.durable(REPORT_DLQ).build(); + } + + + @Bean + public Queue reportResultQueue() { + + return QueueBuilder.durable(REPORT_RESULT_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ) + .build(); + } + + + @Bean + public Queue reportResultDeadLetterQueue() { + + return QueueBuilder.durable(REPORT_RESULT_DLQ).build(); + } + +} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/controller/RedactionReportController.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/controller/RedactionReportController.java deleted file mode 100644 index 7a0a1cc..0000000 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/controller/RedactionReportController.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.iqser.red.service.redaction.report.v1.server.controller; - -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; - -import com.iqser.red.service.redaction.report.v1.api.model.MultiFileRedactionLog; -import com.iqser.red.service.redaction.report.v1.api.model.ReportResult; -import com.iqser.red.service.redaction.report.v1.api.resource.RedactionReportResource; -import com.iqser.red.service.redaction.report.v1.server.service.ReportGenerationService; - -import lombok.RequiredArgsConstructor; - -@RestController -@RequiredArgsConstructor -public class RedactionReportController implements RedactionReportResource { - - private final ReportGenerationService reportGenerationService; - - public ReportResult generateReport(@RequestBody MultiFileRedactionLog multiFileRedactionLog){ - return new ReportResult(reportGenerationService.generateReport(multiFileRedactionLog)); - } - -} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ReportRedactionEntry.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ReportRedactionEntry.java new file mode 100644 index 0000000..4f7df9e --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ReportRedactionEntry.java @@ -0,0 +1,15 @@ +package com.iqser.red.service.redaction.report.v1.server.model; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class ReportRedactionEntry { + + private int page; + private float x; + private float y; + private String section; + private String justification; +} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java new file mode 100644 index 0000000..7c686b0 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java @@ -0,0 +1,98 @@ +package com.iqser.red.service.redaction.report.v1.server.service; + +import java.io.ByteArrayOutputStream; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.poi.xssf.usermodel.XSSFCell; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +@Service +@RequiredArgsConstructor +public class ExcelReportGenerationService { + + public XSSFWorkbook createWorkbook() { + + XSSFWorkbook workbook = new XSSFWorkbook(); + return workbook; + } + + + public XSSFSheet createExcelSheet(XSSFWorkbook workbook) { + + XSSFSheet sheet = workbook.createSheet("Justifications"); + addHeader(sheet); + + return sheet; + } + + + public byte[] generateSingleFileReport(List reportEntries, String filename) { + + XSSFWorkbook workbook = new XSSFWorkbook(); + XSSFSheet sheet = workbook.createSheet("Justifications"); + addHeader(sheet); + + AtomicInteger rowIndex = new AtomicInteger(1); + addEntries(sheet, reportEntries, filename, rowIndex); + + return toByteArray(workbook); + } + + + public void addEntries(XSSFSheet sheet, List reportEntries, String filename, + AtomicInteger rowIndex) { + + reportEntries.forEach(entry -> { + XSSFRow row = sheet.createRow(rowIndex.get()); + XSSFCell filenameCell = row.createCell(0); + filenameCell.setCellValue(filename); + XSSFCell pageColumn = row.createCell(1); + pageColumn.setCellValue(entry.getPage()); + XSSFCell paragraphColumn = row.createCell(2); + paragraphColumn.setCellValue(entry.getSection()); + XSSFCell justificationColumn = row.createCell(3); + justificationColumn.setCellValue(entry.getJustification()); + rowIndex.getAndIncrement(); + }); + } + + + private void addHeader(XSSFSheet sheet) { + + sheet.setColumnWidth(0, 10000); + sheet.setColumnWidth(1, 4000); + sheet.setColumnWidth(2, 20000); + sheet.setColumnWidth(3, 40000); + + XSSFRow row = sheet.createRow(0); + XSSFCell filename = row.createCell(0); + filename.setCellValue("Volume or Document Name"); + XSSFCell page = row.createCell(1); + page.setCellValue("Electronic page No."); + XSSFCell paragraph = row.createCell(2); + paragraph.setCellValue("Paragraph"); + XSSFCell justification = row.createCell(3); + justification.setCellValue("Justification and reference to Article 63 of Regulation (EC) No 1107/2009"); + } + + + @SneakyThrows + public byte[] toByteArray(XSSFWorkbook doc) { + + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + doc.write(byteArrayOutputStream); + doc.close(); + return byteArrayOutputStream.toByteArray(); + } + } + +} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/RedactionLogConverterService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/RedactionLogConverterService.java new file mode 100644 index 0000000..06d3350 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/RedactionLogConverterService.java @@ -0,0 +1,67 @@ +package com.iqser.red.service.redaction.report.v1.server.service; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping; +import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; +import com.iqser.red.service.redaction.v1.model.ManualRedactionType; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.Status; + +@Service +public class RedactionLogConverterService { + + public List convertAndSort(RedactionLog redactionLog, + List legalBasisMappings) { + + List reportEntries = new ArrayList<>(); + + redactionLog.getRedactionLogEntry().forEach(entry -> { + + if (entry.isRedacted()) { + + if (entry.isRecommendation()) { + return; + } + + if (entry.isManual() && entry.getManualRedactionType() + .equals(ManualRedactionType.ADD) && !entry.getStatus().equals(Status.APPROVED)) { + return; + } + + if (entry.isManual() && entry.getManualRedactionType() + .equals(ManualRedactionType.REMOVE) && entry.getStatus().equals(Status.APPROVED)) { + return; + } + + entry.getPositions().forEach(position -> { + reportEntries.add(new ReportRedactionEntry(position.getPage(), position.getTopLeft() + .getX(), position.getTopLeft() + .getY(), entry.getSection(), entry.getLegalBasis() + " " + legalBasisMappings.stream() + .filter(lbm -> lbm.getReason().equalsIgnoreCase(entry.getLegalBasis())) + .findAny() + .map(LegalBasisMapping::getDescription) + .orElse(""))); + }); + } + + }); + + reportEntries.sort((entry1, entry2) -> { + if (entry1.getPage() == entry2.getPage()) { + if (entry1.getY() == entry2.getY()) { + return entry1.getX() < entry2.getX() ? 1 : -1; + } else { + return entry1.getY() < entry2.getY() ? 1 : -1; + } + } + return entry1.getPage() < entry2.getPage() ? -1 : 1; + }); + + return reportEntries; + } + +} \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java index 2f6d5f4..a78f831 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java @@ -1,146 +1,99 @@ package com.iqser.red.service.redaction.report.v1.server.service; -import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping; -import com.iqser.red.service.redaction.report.v1.api.model.MultiFileRedactionLog; -import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient; -import com.iqser.red.service.redaction.report.v1.server.utils.ResourceLoader; -import com.iqser.red.service.redaction.v1.model.ManualRedactionType; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.Status; -import lombok.RequiredArgsConstructor; -import org.apache.poi.xwpf.usermodel.XWPFDocument; -import org.apache.poi.xwpf.usermodel.XWPFParagraph; -import org.apache.poi.xwpf.usermodel.XWPFRun; -import org.apache.poi.xwpf.usermodel.XWPFTable; -import org.apache.poi.xwpf.usermodel.XWPFTableCell; -import org.apache.poi.xwpf.usermodel.XWPFTableRow; -import org.apache.xmlbeans.XmlCursor; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; +import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping; +import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage; +import com.iqser.red.service.redaction.report.v1.api.model.ReportType; +import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; +import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient; +import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; +import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j @Service @RequiredArgsConstructor public class ReportGenerationService { + private final ReportStorageService reportStorageService; + private final ExcelReportGenerationService excelReportService; + private final WordReportGenerationService wordReportGenerationService; private final LegalBasisMappingClient legalBasisMappingClient; - private byte[] efsaTemplate; - private byte[] syngentaTemplate; + private final RedactionLogConverterService redactionLogConverterService; - @PostConstruct - public void init() { + public List generateReport(ReportRequestMessage reportMessage) { - efsaTemplate = ResourceLoader.load("templates/EFSA Template.docx"); - syngentaTemplate = ResourceLoader.load("templates/Syngenta Template.docx"); - } + List legalBasisMappings = legalBasisMappingClient.getLegalBasisMapping(reportMessage.getRuleSetId()); + XSSFWorkbook excelMultiFileWorkbook = null; + XSSFSheet excelMultiFileSheet = null; + AtomicInteger excelRowIndex = new AtomicInteger(1); - public byte[] generateReport(MultiFileRedactionLog multiFileRedactionLog) { - - byte[] template; - - if (multiFileRedactionLog.getTemplate().equals("EFSA Template")) { - template = efsaTemplate; - } else if (multiFileRedactionLog.getTemplate().equals("Syngenta Template")) { - template = syngentaTemplate; - } else { - template = syngentaTemplate; + if (reportMessage.getReportTypes().contains(ReportType.EXCEL_MULTI_FILE)) { + excelMultiFileWorkbook = excelReportService.createWorkbook(); + excelMultiFileSheet = excelReportService.createExcelSheet(excelMultiFileWorkbook); } - try (ByteArrayInputStream is = new ByteArrayInputStream(template)) { - XWPFDocument doc = new XWPFDocument(is); - List legalBasisMappings = legalBasisMappingClient.getLegalBasisMapping(multiFileRedactionLog.getRuleSetId()); - addTableRows(doc, multiFileRedactionLog, legalBasisMappings); - return toByteArray(doc); - } catch (IOException e) { - throw new RuntimeException(e); + List storedFileInformation = new ArrayList<>(); + + int i = 1; + for (String fileId : reportMessage.getFileIds()) { + + long start = System.currentTimeMillis(); + RedactionLog redactionLog = reportStorageService.getRedactionLog(reportMessage.getProjectId(), fileId); + List reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMappings); + + if (reportMessage.getReportTypes().contains(ReportType.EXCEL_MULTI_FILE)) { + excelReportService.addEntries(excelMultiFileSheet, reportEntries, redactionLog.getFilename(), excelRowIndex); + } + + if (reportMessage.getReportTypes().contains(ReportType.EXCEL_SINGLE_FILE)) { + byte[] excelSingleReport = excelReportService.generateSingleFileReport(reportEntries, redactionLog.getFilename()); + String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), excelSingleReport); + storedFileInformation.add(new StoredFileInformation(fileId, redactionLog.getFilename(), storageId, ReportType.EXCEL_SINGLE_FILE)); + } + + if (reportMessage.getReportTypes().contains(ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE)) { + byte[] wordEFSATemplate = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE, reportEntries, redactionLog + .getFilename()); + String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordEFSATemplate); + storedFileInformation.add(new StoredFileInformation(fileId, redactionLog.getFilename(), storageId, ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE)); + } + + if (reportMessage.getReportTypes().contains(ReportType.WORD_SINGLE_FILE_SYNGENTA_TEMPLATE)) { + byte[] wordSyngentaTemplate = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_SYNGENTA_TEMPLATE, reportEntries, redactionLog + .getFilename()); + String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordSyngentaTemplate); + storedFileInformation.add(new StoredFileInformation(fileId, redactionLog.getFilename(), storageId, ReportType.WORD_SINGLE_FILE_SYNGENTA_TEMPLATE)); + } + + long end = System.currentTimeMillis(); + log.info("Successfully processed {}/{} fileIds for downloadId {}, took {}", i, reportMessage.getFileIds() + .size(), reportMessage.getDownloadId(), end - start); + i++; } - } - - private void addTableRows(XWPFDocument doc, MultiFileRedactionLog multiFileRedactionLog, - List legalBasisMappings) { - - XWPFTable table = doc.getTables().get(0); - - XmlCursor cursor = table.getCTTbl().newCursor(); - XWPFParagraph newParagraph = doc.insertNewParagraph(cursor); - XWPFRun run = newParagraph.createRun(); - run.setText("Applied rules: EFSA 1 (Vertebrate Authors)"); - run.setFontSize(10); - - multiFileRedactionLog.getRedactionLogs().stream().sorted(Comparator.comparing(RedactionLog::getFilename)).forEach(fileRedactionLog -> { - - fileRedactionLog.getRedactionLogEntry().forEach(entry -> { - - if (entry.isRedacted()) { - - if (entry.isRecommendation()) { - return; - } - - if (entry.isManual() && entry.getManualRedactionType() - .equals(ManualRedactionType.ADD) && !entry.getStatus().equals(Status.APPROVED)) { - return; - } - - if (entry.isManual() && entry.getManualRedactionType() - .equals(ManualRedactionType.REMOVE) && entry.getStatus().equals(Status.APPROVED)) { - return; - } - - List pages = entry.getPositions() - .stream() - .map(Rectangle::getPage) - .distinct() - .sorted() - .collect(Collectors.toList()); - pages.forEach(page -> { - XWPFTableRow row = table.createRow(); - setText(row.getCell(0), fileRedactionLog.getFilename()); - setText(row.getCell(1), String.valueOf(page)); - setText(row.getCell(2), entry.getSection()); - setText(row.getCell(3), entry.getLegalBasis() + " " + - legalBasisMappings - .stream() - .filter(lbm -> lbm.getReason().equalsIgnoreCase(entry.getLegalBasis())) - .findAny() - .map(LegalBasisMapping::getDescription) - .orElse("")); - }); - } - }); - }); - - } - - - private void setText(XWPFTableCell cell, String value) { - - cell.setVerticalAlignment(XWPFTableCell.XWPFVertAlign.CENTER); - cell.removeParagraph(0); - XWPFParagraph addParagraph = cell.addParagraph(); - XWPFRun run = addParagraph.createRun(); - run.setFontSize(10); - run.setText(value); - } - - - private byte[] toByteArray(XWPFDocument doc) throws IOException { - - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { - doc.write(byteArrayOutputStream); - doc.close(); - return byteArrayOutputStream.toByteArray(); + if (reportMessage.getReportTypes().contains(ReportType.EXCEL_MULTI_FILE)) { + long start = System.currentTimeMillis(); + byte[] multifileExcelReport = excelReportService.toByteArray(excelMultiFileWorkbook); + String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), multifileExcelReport); + storedFileInformation.add(new StoredFileInformation(null, null, storageId, ReportType.EXCEL_MULTI_FILE)); + long end = System.currentTimeMillis(); + log.info("Successfully stored multiFileExcelReport for downloadId {}, took {}", reportMessage.getDownloadId(), end - start); } + + return storedFileInformation; } } \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportMessageReceiver.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportMessageReceiver.java new file mode 100644 index 0000000..433621f --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportMessageReceiver.java @@ -0,0 +1,62 @@ +package com.iqser.red.service.redaction.report.v1.server.service; + +import static com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration.REPORT_QUEUE; +import static com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration.REPORT_RESULT_QUEUE; + +import java.util.List; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage; +import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; +import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@RabbitListener(queues = REPORT_QUEUE) +public class ReportMessageReceiver { + + private final ObjectMapper objectMapper; + private final ReportGenerationService reportGenerationService; + private final RabbitTemplate rabbitTemplate; + + + @RabbitHandler + public void receive(String in) throws JsonProcessingException { + + ReportRequestMessage reportMessage = objectMapper.readValue(in, ReportRequestMessage.class); + + long start = System.currentTimeMillis(); + log.info("Start generating reports for downloadId {}", reportMessage.getDownloadId()); + + List storedFileInformation = reportGenerationService.generateReport(reportMessage); + addToReportResultQueue(reportMessage.getUserId(), reportMessage.getDownloadId(), storedFileInformation, 1); + + long end = System.currentTimeMillis(); + log.info("Successfully generated reports for downloadId {}, took {}", reportMessage.getDownloadId(), end - start); + } + + + private void addToReportResultQueue(String userId, String downloadId, + List storedFileInformation, int priority) { + + try { + rabbitTemplate.convertAndSend(REPORT_RESULT_QUEUE, objectMapper.writeValueAsString(new ReportResultMessage(userId, downloadId, storedFileInformation)), message -> { + message.getMessageProperties().setPriority(priority); + return message; + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + +} \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java new file mode 100644 index 0000000..a0b7a34 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java @@ -0,0 +1,104 @@ +package com.iqser.red.service.redaction.report.v1.server.service; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.apache.poi.xwpf.usermodel.XWPFDocument; +import org.apache.poi.xwpf.usermodel.XWPFParagraph; +import org.apache.poi.xwpf.usermodel.XWPFRun; +import org.apache.poi.xwpf.usermodel.XWPFTable; +import org.apache.poi.xwpf.usermodel.XWPFTableCell; +import org.apache.poi.xwpf.usermodel.XWPFTableRow; +import org.apache.xmlbeans.XmlCursor; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.report.v1.api.model.ReportType; +import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; +import com.iqser.red.service.redaction.report.v1.server.utils.ResourceLoader; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class WordReportGenerationService { + + private byte[] efsaTemplate; + private byte[] syngentaTemplate; + + + @PostConstruct + public void init() { + + efsaTemplate = ResourceLoader.load("templates/EFSA Template.docx"); + syngentaTemplate = ResourceLoader.load("templates/Syngenta Template.docx"); + } + + + public byte[] generateReport(ReportType reportType, List reportEntries, String filename) { + + byte[] template; + + if (reportType.equals(ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE)) { + template = efsaTemplate; + } else if (reportType.equals(ReportType.WORD_SINGLE_FILE_SYNGENTA_TEMPLATE)) { + template = syngentaTemplate; + } else { + template = syngentaTemplate; + } + + try (ByteArrayInputStream is = new ByteArrayInputStream(template)) { + XWPFDocument doc = new XWPFDocument(is); + + addTableRows(doc, reportEntries, filename); + return toByteArray(doc); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + private void addTableRows(XWPFDocument doc, List reportEntries, String filename) { + + XWPFTable table = doc.getTables().get(0); + + XmlCursor cursor = table.getCTTbl().newCursor(); + XWPFParagraph newParagraph = doc.insertNewParagraph(cursor); + XWPFRun run = newParagraph.createRun(); + run.setText("Applied rules: EFSA 1 (Vertebrate Authors)"); + run.setFontSize(10); + + reportEntries.forEach(entry -> { + XWPFTableRow row = table.createRow(); + setText(row.getCell(0), filename); + setText(row.getCell(1), String.valueOf(entry.getPage())); + setText(row.getCell(2), entry.getSection()); + setText(row.getCell(3), entry.getJustification()); + }); + } + + + private void setText(XWPFTableCell cell, String value) { + + cell.setVerticalAlignment(XWPFTableCell.XWPFVertAlign.CENTER); + cell.removeParagraph(0); + XWPFParagraph addParagraph = cell.addParagraph(); + XWPFRun run = addParagraph.createRun(); + run.setFontSize(10); + run.setText(value); + } + + + private byte[] toByteArray(XWPFDocument doc) throws IOException { + + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + doc.write(byteArrayOutputStream); + doc.close(); + return byteArrayOutputStream.toByteArray(); + } + } + +} \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/RedactionReportSettings.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/RedactionReportSettings.java deleted file mode 100644 index 08bc982..0000000 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/RedactionReportSettings.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.iqser.red.service.redaction.report.v1.server.settings; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -import lombok.Data; - -@Data -@ConfigurationProperties("service-template") -public class RedactionReportSettings { - private int numberOfParallelProcessed = 5; -} \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/ReportStorageService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/ReportStorageService.java new file mode 100644 index 0000000..d4f2338 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/ReportStorageService.java @@ -0,0 +1,40 @@ +package com.iqser.red.service.redaction.report.v1.server.storage; + +import java.io.IOException; +import java.util.UUID; + +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.storage.commons.service.StorageService; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class ReportStorageService { + + private final StorageService storageService; + private final ObjectMapper objectMapper; + + + public RedactionLog getRedactionLog(String projectId, String fileId) { + + var redactionLog = storageService.getObject(StorageIdUtils.getRedactionLogStorageId(projectId, fileId)); + try { + return objectMapper.readValue(redactionLog.getInputStream(), RedactionLog.class); + } catch (IOException e) { + throw new RuntimeException("Could not convert RedactionLog", e); + } + } + + + public String storeObject(String downloadId, byte[] data) { + + String storageId = StorageIdUtils.getStorageId(downloadId, UUID.randomUUID().toString()); + storageService.storeObject(storageId, data); + return storageId; + } + +} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/StorageIdUtils.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/StorageIdUtils.java new file mode 100644 index 0000000..2923678 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/StorageIdUtils.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.redaction.report.v1.server.storage; + +public class StorageIdUtils { + + public static String getRedactionLogStorageId(String projectId, String fileId) { + + return projectId + "/" + fileId + ".REDACTION_LOG.json"; + } + + + public static String getStorageId(String downloadId, String tmpFilename) { + + return downloadId + "/" + tmpFilename; + } + +} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java index 4b5a5ef..1d6512f 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java @@ -1,13 +1,11 @@ package com.iqser.red.service.redaction.report.v1.server; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping; -import com.iqser.red.service.redaction.report.v1.api.model.MultiFileRedactionLog; -import com.iqser.red.service.redaction.report.v1.api.model.ReportResult; -import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient; -import com.iqser.red.service.redaction.report.v1.server.controller.RedactionReportController; -import com.iqser.red.service.redaction.v1.model.RedactionLog; +import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; + import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -16,12 +14,19 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit4.SpringRunner; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.List; - -import static org.mockito.Mockito.when; -import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT; +import com.amazonaws.services.s3.AmazonS3; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping; +import com.iqser.red.service.redaction.report.v1.api.model.ReportType; +import com.iqser.red.service.redaction.report.v1.server.client.LegalBasisMappingClient; +import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; +import com.iqser.red.service.redaction.report.v1.server.service.ExcelReportGenerationService; +import com.iqser.red.service.redaction.report.v1.server.service.RedactionLogConverterService; +import com.iqser.red.service.redaction.report.v1.server.service.WordReportGenerationService; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.storage.commons.service.StorageService; @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = RANDOM_PORT) @@ -30,36 +35,74 @@ public class RedactionReportIntegrationTest { @Autowired private ObjectMapper objectMapper; - @Autowired - private RedactionReportController redactionReportController; - @MockBean private LegalBasisMappingClient legalBasisMappingClient; + @MockBean + private StorageService storageService; + + @MockBean + private AmazonS3 s3Client; + + @Autowired + private WordReportGenerationService wordReportGenerationService; + + @Autowired + private ExcelReportGenerationService excelReportGenerationService; + + @MockBean + private MessagingConfiguration messagingConfiguration; + + @Autowired + private RedactionLogConverterService redactionLogConverterService; + @Test - public void testReportGeneration() throws IOException { - - ClassPathResource legalBasisMappingResource = new ClassPathResource("files/LegalBasisMapping.json"); - - List legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() { - }); - - when(legalBasisMappingClient.getLegalBasisMapping("123")).thenReturn(legalBasisMapping); + public void testWordReportGeneration() throws IOException { ClassPathResource redactionLogResource = new ClassPathResource("files/RedactedLog.txt"); RedactionLog redactionLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class); redactionLog.setFilename("TestFile"); - MultiFileRedactionLog multiFileRedactionLog = new MultiFileRedactionLog(List.of(redactionLog), "EFSA Template", "123"); - ReportResult reportResult = redactionReportController.generateReport(multiFileRedactionLog); + ClassPathResource legalBasisMappingResource = new ClassPathResource("files/LegalBasisMapping.json"); - try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/document2.docx")) { - fileOutputStream.write(reportResult.getDocument()); + List legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() { + }); + + List reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping); + + byte[] report = wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE_EFSA_TEMPLATE, reportEntries, redactionLog + .getFilename()); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/efsa_template.docx")) { + fileOutputStream.write(report); } + } + + @Test + public void testExcelReportGeneration() throws IOException { + + ClassPathResource redactionLogResource = new ClassPathResource("files/RedactedLog.txt"); + + RedactionLog redactionLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class); + + redactionLog.setFilename("TestFile"); + + ClassPathResource legalBasisMappingResource = new ClassPathResource("files/LegalBasisMapping.json"); + + List legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() { + }); + + List reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping); + + byte[] report = excelReportGenerationService.generateSingleFileReport(reportEntries, redactionLog.getFilename()); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/report.xlsx")) { + fileOutputStream.write(report); + } } } \ No newline at end of file