diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java index 89e9f2b..1c7321a 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java @@ -29,5 +29,6 @@ public class ExcelModel { private boolean placeholderInFirstRow; private boolean firstRowWritten; private boolean hasScmFunctionPlaceholder; + private boolean hasSkippedPlaceholder; } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java index b78dde3..e90a423 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java @@ -43,6 +43,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import javax.imageio.ImageIO; @@ -99,7 +100,7 @@ public class ExcelReportGenerationService { @Timed("redactmanager_generateExcelReport") - public void generateExcelReport(List reportEntries, + public void generateExcelReport(List allReportEntries, PlaceholderModel placeholderModel, String reportTemplateName, SXSSFWorkbook workbook, @@ -110,6 +111,14 @@ public class ExcelReportGenerationService { long start = System.currentTimeMillis(); + List reportEntries; + + if (!excelModel.isHasSkippedPlaceholder()) { + reportEntries = allReportEntries.stream().filter(entry -> !entry.isSkipped()).collect(Collectors.toList()); + } else { + reportEntries = allReportEntries; + } + try { for (Sheet sheet : workbook) { @@ -235,6 +244,7 @@ public class ExcelReportGenerationService { Map cellsToCopyInPlaceholderRow = new HashMap<>(); boolean hasRssPlaceHolders = false; boolean hasScmFunctionPlaceholder = false; + boolean hasSkippedPlaceholder = false; int placeholderRow = -1; boolean placeholderInFirstRow = false; for (int j = 0; j < sheet.getLastRowNum() + 1; j++) { @@ -260,6 +270,9 @@ public class ExcelReportGenerationService { } if (isRedactionPlaceholder(cellStringValue)) { + if (cellStringValue.equals(SKIPPED_PLACEHOLDER)) { + hasSkippedPlaceholder = true; + } placeholderCellPos.put(i, getFunctionForPlaceHolder(cellStringValue)); placeholderRow = j; cellsToCopyInPlaceholderRow.put(new CellIdentifier(j, i), cell); @@ -292,7 +305,8 @@ public class ExcelReportGenerationService { hasRssPlaceHolders, placeholderInFirstRow, false, - hasScmFunctionPlaceholder); + hasScmFunctionPlaceholder, + hasSkippedPlaceholder); } @@ -376,7 +390,7 @@ public class ExcelReportGenerationService { case REDACTION_VALUE_PLACEHOLDER -> input -> input.getEntry().getValue() != null ? input.getEntry().getValue().replaceAll("\n", " ").replaceAll(" {2,}", " ") : input.getEntry() .getEntityDisplayName(); case REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER -> input -> input.getEntry().getEntityDisplayName(); - case SKIPPED_PLACEHOLDER -> input -> input.getEntry().isSkipped() ? "true" : ""; + case SKIPPED_PLACEHOLDER -> input -> input.getEntry().isSkipped() ? "true" : "false"; default -> input -> ""; }; diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java index 5177464..4ec1356 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java @@ -48,7 +48,7 @@ public class PlaceholderService { public static final String EXCERPT_PLACEHOLDER = "{{redaction.excerpt}}"; public static final String REDACTION_VALUE_PLACEHOLDER = "{{redaction.value}}"; public static final String REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER = "{{redaction.entity.displayName}}"; - public static final String SKIPPED_PLACEHOLDER = "{{redaction.skipped}}"; + public static final String SKIPPED_PLACEHOLDER = "{{redaction.isSkipped}}"; public static final String SCM_FUNCTION_PLACEHOLDER = "{{function.scm}}"; public static final String FILE_ATTRIBUTE_PLACEHOLDER_BASE = "{{file.attribute."; public static final String DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE = "{{dossier.attribute."; diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java index 60c18ac..479ab12 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java @@ -82,7 +82,7 @@ public class WordReportGenerationService { @Timed("redactmanager_generateWordReport") - public int generateWordReport(List reportEntries, + public int generateWordReport(List allReportEntries, PlaceholderModel placeholderModel, String reportTemplateName, XWPFDocument doc, @@ -92,6 +92,8 @@ public class WordReportGenerationService { long start = System.currentTimeMillis(); + List reportEntries = allReportEntries.stream().filter(entry -> !entry.isSkipped()).collect(Collectors.toList()); + int sumOfChars = 0; try { diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java index 89adff4..676115e 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java @@ -148,7 +148,7 @@ public class RedactionReportV2IntegrationTest { .build()); when(fileAttributesConfigClient.getFileAttributeConfigs("dossierTemplateId")).thenReturn(fileAttributeConfig); - var redactionLog = objectMapper.readValue(new ClassPathResource("files/redactionLog.json").getInputStream(), RedactionLog.class); + var redactionLog = objectMapper.readValue(new ClassPathResource("files/65-S10redactionLogWithRedaction.json").getInputStream(), RedactionLog.class); var fileModels = createFileModels(numOfFiles); for (int i = 1; i <= numOfFiles; i++) { when(fileStatusClient.getFileStatus("dossierId", "fileId" + i)).thenReturn(fileModels.get(i - 1)); @@ -216,7 +216,7 @@ public class RedactionReportV2IntegrationTest { @SneakyThrows public void testBasicExcelReportFlow() { - var reportRequestMessage = prepareFlow(1, "templates/Excel Report.xlsx"); + var reportRequestMessage = prepareFlow(2, "templates/Excel QC (incl. Skipped Redactions).xlsx"); processRequest(reportRequestMessage, ".xlsx"); MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateReports", 1, null); diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/65-S10redactionLog.json b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/65-S10redactionLog.json new file mode 100644 index 0000000..094b5f9 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/65-S10redactionLog.json @@ -0,0 +1,289 @@ +{ + "analysisVersion": 1, + "analysisNumber": 1, + "redactionLogEntry": [ + { + "id": "28562647b117cbee1737768475ae3607", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "Text in table", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 143.0472, + "y": 727.91 + }, + "width": 10.452833, + "height": 33.710693, + "page": 1 + } + ], + "sectionNumber": 2, + "textBefore": "Owner (SYN = ", + "textAfter": null, + "comments": null, + "startOffset": 230, + "endOffset": 238, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:51.476409899Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "1f44b16adada98a40be977d224b62d4b", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 546.96716, + "y": 72.89051 + }, + "width": 10.452818, + "height": 33.71469, + "page": 1 + } + ], + "sectionNumber": 5, + "textBefore": "the active substance ", + "textAfter": " – 6", + "comments": null, + "startOffset": 219, + "endOffset": 227, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:51.476423585Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "91ef6d1848e433516662408700de5790", + "type": "hint_only", + "value": "author", + "reason": null, + "matchedRule": 0, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "", + "color": [ + 0.98039216, + 0.59607846, + 0.96862745 + ], + "positions": [ + { + "topLeft": { + "x": 61.0888, + "y": 428.5304 + }, + "width": 11.591162, + "height": 32.84558, + "page": 1 + } + ], + "sectionNumber": 5, + "textBefore": null, + "textAfter": null, + "comments": null, + "startOffset": 147, + "endOffset": 153, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:51.476427502Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": true, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "1c2b8863d0348ec57b337ce2712f0847", + "type": "hint_only", + "value": "author", + "reason": null, + "matchedRule": 0, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "Text in table", + "color": [ + 0.98039216, + 0.59607846, + 0.96862745 + ], + "positions": [ + { + "topLeft": { + "x": 111.60883, + "y": 72.89054 + }, + "width": 11.591154, + "height": 32.80046, + "page": 1 + } + ], + "sectionNumber": 2, + "textBefore": null, + "textAfter": null, + "comments": null, + "startOffset": 0, + "endOffset": 6, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:51.476432602Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": true, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + } + ], + "legalBasis": [ + { + "name": "1.1 personal data (incl. geolocation); Article 39(e)(3)", + "description": "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)", + "reason": "Article 39(e)(3) of Regulation (EC) No 178/2002" + }, + { + "name": "1.2 vertebrate study related personal data (incl. geolocation); Article 39(e)(2)", + "description": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information", + "reason": "Article 39(e)(2) of Regulation (EC) No 178/2002" + }, + { + "name": "2. manufacturing or production process", + "description": "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "3. links between a producer and applicant", + "description": "commercial links between a producer or importer and the applicant or the authorisation holder, where applicable", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "4. commercial information", + "description": "commercial information revealing sourcing, market shares or business strategy of the applicant", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "5. quantitative composition", + "description": "quantitative composition of the subject matter of the request, except for information which is relevant to the assessment of safety", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "6. specification of impurity", + "description": "the specification of impurity of the active substance and the related methods of analysis for impurities in the active substance as manufactured, except for the impurities that are considered to be toxicologically, ecotoxicologically or environmentally relevant and the related methods of analysis for such impurities", + "reason": "Article 63(2)(b) of Regulation (EC) No 1107/2009" + }, + { + "name": "7. results of production batches", + "description": "results of production batches of the active substance including impurities", + "reason": "Article 63(2)(c) of Regulation (EC) No 1107/2009" + }, + { + "name": "8. composition of a plant protection product", + "description": "information on the complete composition of a plant protection product", + "reason": "Article 63(2)(d) of Regulation (EC) No 1107/2009" + } + ], + "dictionaryVersion": 18, + "dossierDictionaryVersion": 1, + "rulesVersion": 1, + "legalBasisVersion": 2 +} \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/65-S10redactionLogWithRedaction.json b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/65-S10redactionLogWithRedaction.json new file mode 100644 index 0000000..2786c63 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/65-S10redactionLogWithRedaction.json @@ -0,0 +1,355 @@ +{ + "analysisVersion": 1, + "analysisNumber": 1, + "redactionLogEntry": [ + { + "id": "28562647b117cbee1737768475ae3607", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "Text in table", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 143.0472, + "y": 727.91 + }, + "width": 10.452833, + "height": 33.710693, + "page": 1 + } + ], + "sectionNumber": 2, + "textBefore": "Owner (SYN = ", + "textAfter": null, + "comments": null, + "startOffset": 230, + "endOffset": 238, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:51.476409899Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "1f44b16adada98a40be977d224b62d4b", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 546.96716, + "y": 72.89051 + }, + "width": 10.452818, + "height": 33.71469, + "page": 1 + } + ], + "sectionNumber": 5, + "textBefore": "the active substance ", + "textAfter": " – 6", + "comments": null, + "startOffset": 219, + "endOffset": 227, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:51.476423585Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "91ef6d1848e433516662408700de5790", + "type": "hint_only", + "value": "author", + "reason": null, + "matchedRule": 0, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "", + "color": [ + 0.98039216, + 0.59607846, + 0.96862745 + ], + "positions": [ + { + "topLeft": { + "x": 61.0888, + "y": 428.5304 + }, + "width": 11.591162, + "height": 32.84558, + "page": 1 + } + ], + "sectionNumber": 5, + "textBefore": null, + "textAfter": null, + "comments": null, + "startOffset": 147, + "endOffset": 153, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:51.476427502Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": true, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "1c2b8863d0348ec57b337ce2712f0847", + "type": "hint_only", + "value": "author", + "reason": null, + "matchedRule": 0, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "Text in table", + "color": [ + 0.98039216, + 0.59607846, + 0.96862745 + ], + "positions": [ + { + "topLeft": { + "x": 111.60883, + "y": 72.89054 + }, + "width": 11.591154, + "height": 32.80046, + "page": 1 + } + ], + "sectionNumber": 2, + "textBefore": null, + "textAfter": null, + "comments": null, + "startOffset": 0, + "endOffset": 6, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:51.476432602Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": true, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "b1372ac26ef7c9ac846636dc90ddcf05", + "type": "manual", + "value": "Published", + "reason": "quantitative composition of the subject matter of the request, except for information which is relevant to the assessment of safety", + "matchedRule": 0, + "rectangle": false, + "legalBasis": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)", + "imported": false, + "redacted": true, + "section": "Text in table", + "color": [ + 0.5764706, + 0.59607846, + 0.627451 + ], + "positions": [ + { + "topLeft": { + "x": 157.0195, + "y": 350.69205 + }, + "width": 13.96788, + "height": -47.821453, + "page": 1 + } + ], + "sectionNumber": 2, + "textBefore": "", + "textAfter": "", + "comments": null, + "startOffset": 0, + "endOffset": 0, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 2, + "type": "ADDED", + "dateTime": "2023-03-20T11:27:09.936Z" + } + ], + "manualChanges": [ + { + "annotationStatus": "APPROVED", + "manualRedactionType": "ADD_LOCALLY", + "processedDate": "2023-03-20T11:27:09.944Z", + "requestedDate": "2023-03-20T11:27:09.936Z", + "userId": "98755882-a216-44e6-92c1-0c191ad134ce", + "propertyChanges": {}, + "processed": true + } + ], + "engines": null, + "reference": null, + "importedRedactionIntersections": [], + "localManualRedaction": true, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": false, + "dossierDictionaryEntry": false + } + ], + "legalBasis": [ + { + "name": "1.1 personal data (incl. geolocation); Article 39(e)(3)", + "description": "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)", + "reason": "Article 39(e)(3) of Regulation (EC) No 178/2002" + }, + { + "name": "1.2 vertebrate study related personal data (incl. geolocation); Article 39(e)(2)", + "description": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information", + "reason": "Article 39(e)(2) of Regulation (EC) No 178/2002" + }, + { + "name": "2. manufacturing or production process", + "description": "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "3. links between a producer and applicant", + "description": "commercial links between a producer or importer and the applicant or the authorisation holder, where applicable", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "4. commercial information", + "description": "commercial information revealing sourcing, market shares or business strategy of the applicant", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "5. quantitative composition", + "description": "quantitative composition of the subject matter of the request, except for information which is relevant to the assessment of safety", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "6. specification of impurity", + "description": "the specification of impurity of the active substance and the related methods of analysis for impurities in the active substance as manufactured, except for the impurities that are considered to be toxicologically, ecotoxicologically or environmentally relevant and the related methods of analysis for such impurities", + "reason": "Article 63(2)(b) of Regulation (EC) No 1107/2009" + }, + { + "name": "7. results of production batches", + "description": "results of production batches of the active substance including impurities", + "reason": "Article 63(2)(c) of Regulation (EC) No 1107/2009" + }, + { + "name": "8. composition of a plant protection product", + "description": "information on the complete composition of a plant protection product", + "reason": "Article 63(2)(d) of Regulation (EC) No 1107/2009" + } + ], + "dictionaryVersion": 18, + "dossierDictionaryVersion": 1, + "rulesVersion": 1, + "legalBasisVersion": 2 +} \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/67-S6redactionLog.json b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/67-S6redactionLog.json new file mode 100644 index 0000000..3dca638 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/files/67-S6redactionLog.json @@ -0,0 +1,639 @@ +{ + "analysisVersion": 1, + "analysisNumber": 1, + "redactionLogEntry": [ + { + "id": "ac54224bab3fa173a28c1fc943abcb2b", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "Text in table", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 143.0472, + "y": 727.85 + }, + "width": 10.452833, + "height": 33.710693, + "page": 1 + } + ], + "sectionNumber": 2, + "textBefore": "Owner (SYN = ", + "textAfter": null, + "comments": null, + "startOffset": 230, + "endOffset": 238, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477812432Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "badbdba336f4d754024bb1f820a61b0a", + "type": "CBI_author", + "value": "Green R.", + "reason": "Author found", + "matchedRule": 10, + "rectangle": false, + "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "imported": false, + "redacted": true, + "section": "Text in table", + "color": [ + 0.5764706, + 0.59607846, + 0.627451 + ], + "positions": [ + { + "topLeft": { + "x": 253.67392, + "y": 162.84514 + }, + "width": 11.54608, + "height": 36.198288, + "page": 1 + } + ], + "sectionNumber": 6, + "textBefore": null, + "textAfter": null, + "comments": null, + "startOffset": 15, + "endOffset": 23, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477821319Z" + } + ], + "manualChanges": [], + "engines": [ + "NER", + "DICTIONARY", + "RULE" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "0cbb552705d3294bf579dcfa97ea6766", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 290.2385, + "y": 644.72015 + }, + "width": 10.981504, + "height": 33.48163, + "page": 1 + } + ], + "sectionNumber": 9, + "textBefore": "providing access to ", + "textAfter": " specific know-how", + "comments": null, + "startOffset": 393, + "endOffset": 401, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477826438Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "c688622f3544cd4d9c4aafef2c77f15d", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "Text in table", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 237.65378, + "y": 302.87067 + }, + "width": 11.546049, + "height": 37.251373, + "page": 1 + } + ], + "sectionNumber": 5, + "textBefore": "to Document J ", + "textAfter": " File No", + "comments": null, + "startOffset": 102, + "endOffset": 110, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477828843Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "8ac1b192ab6ca20375149f01a8ce7334", + "type": "CBI_author", + "value": "Akkan Z., Botham J.", + "reason": "Author found", + "matchedRule": 10, + "rectangle": false, + "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "imported": false, + "redacted": true, + "section": "Text in table", + "color": [ + 0.5764706, + 0.59607846, + 0.627451 + ], + "positions": [ + { + "topLeft": { + "x": 214.67392, + "y": 162.85521 + }, + "width": 11.546062, + "height": 83.2892, + "page": 1 + } + ], + "sectionNumber": 5, + "textBefore": null, + "textAfter": null, + "comments": null, + "startOffset": 15, + "endOffset": 34, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477830937Z" + } + ], + "manualChanges": [], + "engines": [ + "RULE" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": false, + "dossierDictionaryEntry": false + }, + { + "id": "a6a7198a8089e8e06107af5f6a2d1e73", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 290.2385, + "y": 75.3515 + }, + "width": 10.981504, + "height": 33.536697, + "page": 1 + } + ], + "sectionNumber": 9, + "textBefore": "2013 ZA1296_10191 *", + "textAfter": " requests data", + "comments": null, + "startOffset": 237, + "endOffset": 245, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477833081Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "2a0f1f825a91671322b6c7adef385367", + "type": "hint_only", + "value": "author", + "reason": null, + "matchedRule": 0, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "Text in table", + "color": [ + 0.98039216, + 0.59607846, + 0.96862745 + ], + "positions": [ + { + "topLeft": { + "x": 111.60883, + "y": 162.83318 + }, + "width": 11.591154, + "height": 32.800476, + "page": 1 + } + ], + "sectionNumber": 2, + "textBefore": null, + "textAfter": null, + "comments": null, + "startOffset": 11, + "endOffset": 17, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477835365Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": true, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "a22c85dd1d9005345e48e1e573745567", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 546.96716, + "y": 72.830505 + }, + "width": 10.452818, + "height": 33.714684, + "page": 1 + } + ], + "sectionNumber": 9, + "textBefore": "the active substance ", + "textAfter": " – 6", + "comments": null, + "startOffset": 196, + "endOffset": 204, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477837359Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "181eec8da7dff594abd8181370a4b2b5", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 290.2385, + "y": 442.6209 + }, + "width": 10.981504, + "height": 33.48181, + "page": 1 + } + ], + "sectionNumber": 9, + "textBefore": "information might undermine ", + "textAfter": "’s commercial interests", + "comments": null, + "startOffset": 338, + "endOffset": 346, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477839473Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + }, + { + "id": "b907ba9ef5993bd3d12d2098cb03759e", + "type": "CBI_address", + "value": "Syngenta", + "reason": "Address found for non vertebrate study", + "matchedRule": 3, + "rectangle": false, + "legalBasis": null, + "imported": false, + "redacted": false, + "section": "Text in table", + "color": [ + 0.76862746, + 0.59607846, + 0.98039216 + ], + "positions": [ + { + "topLeft": { + "x": 276.65378, + "y": 302.87067 + }, + "width": 11.54608, + "height": 37.251373, + "page": 1 + } + ], + "sectionNumber": 6, + "textBefore": "to Document J ", + "textAfter": " File No", + "comments": null, + "startOffset": 91, + "endOffset": 99, + "imageHasTransparency": false, + "excluded": false, + "sourceId": null, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2023-02-28T10:05:46.477841627Z" + } + ], + "manualChanges": [], + "engines": [ + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "localManualRedaction": false, + "manuallyRemoved": false, + "hint": false, + "recommendation": false, + "falsePositive": false, + "image": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false + } + ], + "legalBasis": [ + { + "name": "1.1 personal data (incl. geolocation); Article 39(e)(3)", + "description": "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)", + "reason": "Article 39(e)(3) of Regulation (EC) No 178/2002" + }, + { + "name": "1.2 vertebrate study related personal data (incl. geolocation); Article 39(e)(2)", + "description": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information", + "reason": "Article 39(e)(2) of Regulation (EC) No 178/2002" + }, + { + "name": "2. manufacturing or production process", + "description": "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "3. links between a producer and applicant", + "description": "commercial links between a producer or importer and the applicant or the authorisation holder, where applicable", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "4. commercial information", + "description": "commercial information revealing sourcing, market shares or business strategy of the applicant", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "5. quantitative composition", + "description": "quantitative composition of the subject matter of the request, except for information which is relevant to the assessment of safety", + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + }, + { + "name": "6. specification of impurity", + "description": "the specification of impurity of the active substance and the related methods of analysis for impurities in the active substance as manufactured, except for the impurities that are considered to be toxicologically, ecotoxicologically or environmentally relevant and the related methods of analysis for such impurities", + "reason": "Article 63(2)(b) of Regulation (EC) No 1107/2009" + }, + { + "name": "7. results of production batches", + "description": "results of production batches of the active substance including impurities", + "reason": "Article 63(2)(c) of Regulation (EC) No 1107/2009" + }, + { + "name": "8. composition of a plant protection product", + "description": "information on the complete composition of a plant protection product", + "reason": "Article 63(2)(d) of Regulation (EC) No 1107/2009" + } + ], + "dictionaryVersion": 18, + "dossierDictionaryVersion": 1, + "rulesVersion": 1, + "legalBasisVersion": 2 +} \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/templates/Excel QC (incl. Skipped Redactions).xlsx b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/templates/Excel QC (incl. Skipped Redactions).xlsx new file mode 100644 index 0000000..e556a18 Binary files /dev/null and b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/templates/Excel QC (incl. Skipped Redactions).xlsx differ