From a8eac995171fac59c1bc18b9ec534b613b3fb82f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Mon, 18 Dec 2023 11:00:22 +0100 Subject: [PATCH] RED-7946: Truncate extreme long sections and values for report --- .../service/EntityLogConverterService.java | 60 +++++++++++++++---- .../settings/ReportTemplateSettings.java | 2 + .../RedactionReportIntegrationTest.java | 27 +++++---- .../EntityLogConverterServiceTest.java | 16 +++-- 4 files changed, 76 insertions(+), 29 deletions(-) diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterService.java index b0af077..4831bb7 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterService.java @@ -10,6 +10,7 @@ import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -29,12 +30,15 @@ import com.iqser.red.service.redaction.report.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.report.v1.server.client.DossierClient; import com.iqser.red.service.redaction.report.v1.server.client.EntityLogClient; import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; +import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings; import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +@Slf4j @Service @RequiredArgsConstructor public class EntityLogConverterService { @@ -47,6 +51,8 @@ public class EntityLogConverterService { private final EntityLogClient entityLogClient; + private final ReportTemplateSettings settings; + @Timed("redactmanager_getReportEntries") public List getReportEntries(String dossierId, String fileId, boolean isExcluded, boolean includeUnprocessed) { @@ -68,7 +74,7 @@ public class EntityLogConverterService { return new ArrayList<>(); } - return convertAndSort(entityLog, entityLog.getLegalBasis(), mapOfEntityDisplayName); + return convertAndSort(dossierId, fileId, entityLog, entityLog.getLegalBasis(), mapOfEntityDisplayName); } @@ -89,15 +95,16 @@ public class EntityLogConverterService { } - public List convertAndSort(EntityLog entityLog, + public List convertAndSort(String dossierId, + String fileId, + EntityLog entityLog, List legalBasisMappings, Map mapOfEntityDisplayName) { List reportEntries = new ArrayList<>(); entityLog.getEntityLogEntry().forEach(entry -> { - var isSkipped = entry.getState() != EntryState.APPLIED - && !(entry.getEntryType().equals(EntryType.HINT) || entry.getState().equals(EntryState.IGNORED)); + var isSkipped = entry.getState() != EntryState.APPLIED && !(entry.getEntryType().equals(EntryType.HINT) || entry.getState().equals(EntryState.IGNORED)); if (entry.getState() == EntryState.APPLIED || isSkipped) { @@ -113,21 +120,44 @@ public class EntityLogConverterService { return; } - if (entry.getEntryType() == EntryType.FALSE_POSITIVE ) { + if (entry.getEntryType() == EntryType.FALSE_POSITIVE) { return; } - if (!entry.getManualChanges().isEmpty() - && (entry.getEntryType() == EntryType.IMAGE || entry.getEntryType() == EntryType.IMAGE_HINT) - && entry.getManualChanges().get(entry.getManualChanges().size() - 1).getProcessedDate() == null - && entry.getManualChanges().get(entry.getManualChanges().size() - 1).getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE)) { + if (settings.getMaxRedactionEntryValueLength() > 0 && entry.getValue() != null && entry.getValue().length() > settings.getMaxRedactionEntryValueLength()) { + entry.setValue(entry.getValue().substring(0, settings.getMaxRedactionEntryValueLength()) + "..."); + log.warn("Truncated value in dossier {}, file {} on pages {} to {} chars", + dossierId, + fileId, + entry.getPositions().stream().map(Position::getPageNumber).collect(Collectors.toList()), + settings.getMaxRedactionEntryValueLength()); + } + + if (settings.getMaxSectionLength() > 0 && entry.getSection() != null && entry.getSection().length() > settings.getMaxSectionLength()){ + entry.setSection(entry.getSection().substring(0, settings.getMaxSectionLength()) + "..."); + log.warn("Truncated section in dossier {}, file {} on pages {} to {} chars", + dossierId, + fileId, + entry.getPositions().stream().map(Position::getPageNumber).collect(Collectors.toList()), + settings.getMaxSectionLength()); + } + + + if (!entry.getManualChanges().isEmpty() && (entry.getEntryType() == EntryType.IMAGE || entry.getEntryType() == EntryType.IMAGE_HINT) && entry.getManualChanges() + .get(entry.getManualChanges().size() - 1) + .getProcessedDate() == null && entry.getManualChanges() + .get(entry.getManualChanges().size() - 1) + .getManualRedactionType() + .equals(ManualRedactionType.RECATEGORIZE)) { return; } - if (!entry.getManualChanges().isEmpty() - && (entry.isDictionaryEntry() || entry.isDossierDictionaryEntry()) - && entry.getManualChanges().get(entry.getManualChanges().size() - 1).getProcessedDate() == null - && entry.getManualChanges().get(entry.getManualChanges().size() - 1).getManualRedactionType().equals(ManualRedactionType.ADD_TO_DICTIONARY)) { + if (!entry.getManualChanges().isEmpty() && (entry.isDictionaryEntry() || entry.isDossierDictionaryEntry()) && entry.getManualChanges() + .get(entry.getManualChanges().size() - 1) + .getProcessedDate() == null && entry.getManualChanges() + .get(entry.getManualChanges().size() - 1) + .getManualRedactionType() + .equals(ManualRedactionType.ADD_TO_DICTIONARY)) { return; } @@ -174,6 +204,7 @@ public class EntityLogConverterService { return reportEntries; } + private String determineValue(EntityLogEntry entry) { if (entry.getManualChanges() != null && !entry.getManualChanges().isEmpty()) { @@ -231,13 +262,16 @@ public class EntityLogConverterService { } + private boolean lastChangeIsRemoved(List changes) { return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false); } + private Optional last(List list) { return list == null || list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1)); } + } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java index 712e49c..742ae9e 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java @@ -9,5 +9,7 @@ import lombok.Data; public class ReportTemplateSettings { private int multiFileChunkSize = 1000000; + private int maxRedactionEntryValueLength = 1000; + private int maxSectionLength = 100; } \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java index 36ca190..c1bc93e 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java @@ -83,6 +83,9 @@ import lombok.SneakyThrows; @SuppressWarnings("PMD") public class RedactionReportIntegrationTest { + private static final String DOSSIER_ID = "DossierId"; + private static final String FILE_ID = "FileId"; + @MockBean TenantsClient tenantsClient; @@ -158,7 +161,7 @@ public class RedactionReportIntegrationTest { EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class); List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>()); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>()); var wordTemplateResource = new ClassPathResource("templates/Justification Appendix A1.docx"); var imageResource = new ClassPathResource("files/exampleImage.jpg"); @@ -195,7 +198,7 @@ public class RedactionReportIntegrationTest { EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class); List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>()); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>()); var wordTemplateResource = new ClassPathResource("templates/Justification Appendix A2.docx"); var imageResource = new ClassPathResource("files/exampleImage.jpg"); @@ -273,7 +276,7 @@ public class RedactionReportIntegrationTest { EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class); List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>()); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>()); ClassPathResource templateResource = new ClassPathResource("templates/IUCLID_Template.docx"); XWPFDocument doc = new XWPFDocument(templateResource.getInputStream()); @@ -298,7 +301,7 @@ public class RedactionReportIntegrationTest { EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class); List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>()); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>()); ClassPathResource templateResource = new ClassPathResource("templates/Seeds - New Justification Form.docx"); XWPFDocument doc = new XWPFDocument(templateResource.getInputStream()); @@ -324,11 +327,11 @@ public class RedactionReportIntegrationTest { EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class); List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>()); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>()); FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build(); EntityLog entityLogSecondFile = objectMapper.readValue(new ClassPathResource("files/entityLogWithManualRedactions.json").getInputStream(), EntityLog.class); - List reportEntriesSecondFile = entityLogConverterService.convertAndSort(entityLogSecondFile, legalBasisMapping, new HashMap<>()); + List reportEntriesSecondFile = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLogSecondFile, legalBasisMapping, new HashMap<>()); ClassPathResource templateResource = new ClassPathResource("templates/Seeds-NewJustificationForm.docx"); XWPFDocument doc = new XWPFDocument(templateResource.getInputStream()); @@ -355,7 +358,7 @@ public class RedactionReportIntegrationTest { EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class); List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>()); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>()); var wordTemplateResource = new ClassPathResource("templates/6464 appendix_b EFSA dRAR justification.docx"); var imageResource = new ClassPathResource("files/exampleImage.jpg"); @@ -393,7 +396,7 @@ public class RedactionReportIntegrationTest { List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); Map mapOfEntityDisplayName = createEntityDisplayNames(entityLog); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, mapOfEntityDisplayName); ClassPathResource templateResource = new ClassPathResource("templates/Excel Report.xlsx"); @@ -422,7 +425,7 @@ public class RedactionReportIntegrationTest { List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); Map mapOfEntityDisplayName = createEntityDisplayNames(entityLog); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID,entityLog, legalBasisMapping, mapOfEntityDisplayName); FileModel fileModel = FileModel.builder().filename("filename").build(); @@ -437,7 +440,7 @@ public class RedactionReportIntegrationTest { excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName", fileModel, excelModel, false); EntityLog entityLogSecondFile = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class); - List reportEntriesSecondFile = entityLogConverterService.convertAndSort(entityLogSecondFile, legalBasisMapping, mapOfEntityDisplayName); + List reportEntriesSecondFile = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID,entityLogSecondFile, legalBasisMapping, mapOfEntityDisplayName); FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build(); excelTemplateReportGenerationService.generateExcelReport(reportEntriesSecondFile, placeholders, @@ -464,7 +467,7 @@ public class RedactionReportIntegrationTest { List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); Map mapOfEntityDisplayName = createEntityDisplayNames(entityLog); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID,entityLog, legalBasisMapping, mapOfEntityDisplayName); var imageResource = new ClassPathResource("files/exampleImage.jpg"); FileModel fileModel = FileModel.builder().filename("filename").build(); @@ -507,7 +510,7 @@ public class RedactionReportIntegrationTest { List fileAttributeConfigs = objectMapper.readValue(new ClassPathResource("files/scm/fileAttributes.json").getInputStream(), new TypeReference<>() { }); Map mapOfEntityDisplayName = createEntityDisplayNames(entityLog); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, mapOfEntityDisplayName); when(componentClient.getComponentLog(dossier.getDossierId(), fileModel.getId(), true)).thenReturn(componentLog); when(fileAttributesConfigClient.getFileAttributeConfigs(dossier.getDossierTemplateId())).thenReturn(fileAttributeConfigs); diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterServiceTest.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterServiceTest.java index 1c39cbd..a0645f0 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterServiceTest.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterServiceTest.java @@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.report.v1.server.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import java.util.HashMap; @@ -13,6 +11,7 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; +import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -25,11 +24,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.redaction.report.v1.server.Application; import com.iqser.red.service.redaction.report.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.report.v1.server.client.DossierClient; import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; +import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings; import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.knecon.fforesight.tenantcommons.TenantsClient; @@ -41,6 +40,9 @@ import lombok.SneakyThrows; @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) public class EntityLogConverterServiceTest { + private static final String DOSSIER_ID = "DossierId"; + private static final String FILE_ID = "FileId"; + @MockBean TenantsClient tenantsClient; @@ -53,12 +55,16 @@ public class EntityLogConverterServiceTest { @MockBean private ReportStorageService reportStorageService; + @Mock + private ReportTemplateSettings settings; + @InjectMocks private EntityLogConverterService entityLogConverterService; @Autowired private ObjectMapper objectMapper; + @Test @SneakyThrows public void testConvertAndSort() { @@ -66,7 +72,9 @@ public class EntityLogConverterServiceTest { EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class); List legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() { }); - List reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>()); + when(settings.getMaxRedactionEntryValueLength()).thenReturn(1000); + when(settings.getMaxSectionLength()).thenReturn(100); + List reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>()); assertNotNull(reportEntries); assertFalse(reportEntries.isEmpty());