Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

RED-7946: Truncate extreme long sections and values for report

Browse Source
This commit is contained in:
Dominique Eifländer 2023-12-18 11:00:22 +01:00
parent 10a1e95257
commit a8eac99517
4 changed files with 76 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterService.java
View File

@ -10,6 +10,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@ -29,12 +30,15 @@ import com.iqser.red.service.redaction.report.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierClient;
import com.iqser.red.service.redaction.report.v1.server.client.EntityLogClient;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class EntityLogConverterService {
@ -47,6 +51,8 @@ public class EntityLogConverterService {
private final EntityLogClient entityLogClient;
private final ReportTemplateSettings settings;
@Timed("redactmanager_getReportEntries")
public List<ReportRedactionEntry> getReportEntries(String dossierId, String fileId, boolean isExcluded, boolean includeUnprocessed) {
@ -68,7 +74,7 @@ public class EntityLogConverterService {
return new ArrayList<>();
}
return convertAndSort(entityLog, entityLog.getLegalBasis(), mapOfEntityDisplayName);
return convertAndSort(dossierId, fileId, entityLog, entityLog.getLegalBasis(), mapOfEntityDisplayName);
}
@ -89,15 +95,16 @@ public class EntityLogConverterService {
}
public List<ReportRedactionEntry> convertAndSort(EntityLog entityLog,
public List<ReportRedactionEntry> convertAndSort(String dossierId,
String fileId,
EntityLog entityLog,
List<EntityLogLegalBasis> legalBasisMappings,
Map<String, String> mapOfEntityDisplayName) {
List<ReportRedactionEntry> reportEntries = new ArrayList<>();
entityLog.getEntityLogEntry().forEach(entry -> {
var isSkipped = entry.getState() != EntryState.APPLIED
&& !(entry.getEntryType().equals(EntryType.HINT) || entry.getState().equals(EntryState.IGNORED));
var isSkipped = entry.getState() != EntryState.APPLIED && !(entry.getEntryType().equals(EntryType.HINT) || entry.getState().equals(EntryState.IGNORED));
if (entry.getState() == EntryState.APPLIED || isSkipped) {
@ -113,21 +120,44 @@ public class EntityLogConverterService {
return;
}
if (entry.getEntryType() == EntryType.FALSE_POSITIVE ) {
if (entry.getEntryType() == EntryType.FALSE_POSITIVE) {
return;
}
if (!entry.getManualChanges().isEmpty()
&& (entry.getEntryType() == EntryType.IMAGE || entry.getEntryType() == EntryType.IMAGE_HINT)
&& entry.getManualChanges().get(entry.getManualChanges().size() - 1).getProcessedDate() == null
&& entry.getManualChanges().get(entry.getManualChanges().size() - 1).getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE)) {
if (settings.getMaxRedactionEntryValueLength() > 0 && entry.getValue() != null && entry.getValue().length() > settings.getMaxRedactionEntryValueLength()) {
entry.setValue(entry.getValue().substring(0, settings.getMaxRedactionEntryValueLength()) + "...");
log.warn("Truncated value in dossier {}, file {} on pages {} to {} chars",
dossierId,
fileId,
entry.getPositions().stream().map(Position::getPageNumber).collect(Collectors.toList()),
settings.getMaxRedactionEntryValueLength());
}
if (settings.getMaxSectionLength() > 0 && entry.getSection() != null && entry.getSection().length() > settings.getMaxSectionLength()){
entry.setSection(entry.getSection().substring(0, settings.getMaxSectionLength()) + "...");
log.warn("Truncated section in dossier {}, file {} on pages {} to {} chars",
dossierId,
fileId,
entry.getPositions().stream().map(Position::getPageNumber).collect(Collectors.toList()),
settings.getMaxSectionLength());
}
if (!entry.getManualChanges().isEmpty() && (entry.getEntryType() == EntryType.IMAGE || entry.getEntryType() == EntryType.IMAGE_HINT) && entry.getManualChanges()
.get(entry.getManualChanges().size() - 1)
.getProcessedDate() == null && entry.getManualChanges()
.get(entry.getManualChanges().size() - 1)
.getManualRedactionType()
.equals(ManualRedactionType.RECATEGORIZE)) {
return;
}
if (!entry.getManualChanges().isEmpty()
&& (entry.isDictionaryEntry() || entry.isDossierDictionaryEntry())
&& entry.getManualChanges().get(entry.getManualChanges().size() - 1).getProcessedDate() == null
&& entry.getManualChanges().get(entry.getManualChanges().size() - 1).getManualRedactionType().equals(ManualRedactionType.ADD_TO_DICTIONARY)) {
if (!entry.getManualChanges().isEmpty() && (entry.isDictionaryEntry() || entry.isDossierDictionaryEntry()) && entry.getManualChanges()
.get(entry.getManualChanges().size() - 1)
.getProcessedDate() == null && entry.getManualChanges()
.get(entry.getManualChanges().size() - 1)
.getManualRedactionType()
.equals(ManualRedactionType.ADD_TO_DICTIONARY)) {
return;
}
@ -174,6 +204,7 @@ public class EntityLogConverterService {
return reportEntries;
}
private String determineValue(EntityLogEntry entry) {
if (entry.getManualChanges() != null && !entry.getManualChanges().isEmpty()) {
@ -231,13 +262,16 @@ public class EntityLogConverterService {
}
private boolean lastChangeIsRemoved(List<Change> changes) {
return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
}
private <T> Optional<T> last(List<T> list) {
return list == null || list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
}
}

2
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java
View File

@ -9,5 +9,7 @@ import lombok.Data;
public class ReportTemplateSettings {
private int multiFileChunkSize = 1000000;
private int maxRedactionEntryValueLength = 1000;
private int maxSectionLength = 100;
}

27
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java
View File

@ -83,6 +83,9 @@ import lombok.SneakyThrows;
@SuppressWarnings("PMD")
public class RedactionReportIntegrationTest {
private static final String DOSSIER_ID = "DossierId";
private static final String FILE_ID = "FileId";
@MockBean
TenantsClient tenantsClient;
@ -158,7 +161,7 @@ public class RedactionReportIntegrationTest {
EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>());
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>());
var wordTemplateResource = new ClassPathResource("templates/Justification Appendix A1.docx");
var imageResource = new ClassPathResource("files/exampleImage.jpg");
@ -195,7 +198,7 @@ public class RedactionReportIntegrationTest {
EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>());
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>());
var wordTemplateResource = new ClassPathResource("templates/Justification Appendix A2.docx");
var imageResource = new ClassPathResource("files/exampleImage.jpg");
@ -273,7 +276,7 @@ public class RedactionReportIntegrationTest {
EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>());
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>());
ClassPathResource templateResource = new ClassPathResource("templates/IUCLID_Template.docx");
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
@ -298,7 +301,7 @@ public class RedactionReportIntegrationTest {
EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>());
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>());
ClassPathResource templateResource = new ClassPathResource("templates/Seeds - New Justification Form.docx");
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
@ -324,11 +327,11 @@ public class RedactionReportIntegrationTest {
EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>());
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>());
FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build();
EntityLog entityLogSecondFile = objectMapper.readValue(new ClassPathResource("files/entityLogWithManualRedactions.json").getInputStream(), EntityLog.class);
List<ReportRedactionEntry> reportEntriesSecondFile = entityLogConverterService.convertAndSort(entityLogSecondFile, legalBasisMapping, new HashMap<>());
List<ReportRedactionEntry> reportEntriesSecondFile = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLogSecondFile, legalBasisMapping, new HashMap<>());
ClassPathResource templateResource = new ClassPathResource("templates/Seeds-NewJustificationForm.docx");
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
@ -355,7 +358,7 @@ public class RedactionReportIntegrationTest {
EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>());
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>());
var wordTemplateResource = new ClassPathResource("templates/6464 appendix_b EFSA dRAR justification.docx");
var imageResource = new ClassPathResource("files/exampleImage.jpg");
@ -393,7 +396,7 @@ public class RedactionReportIntegrationTest {
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
Map<String, String> mapOfEntityDisplayName = createEntityDisplayNames(entityLog);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, mapOfEntityDisplayName);
ClassPathResource templateResource = new ClassPathResource("templates/Excel Report.xlsx");
@ -422,7 +425,7 @@ public class RedactionReportIntegrationTest {
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
Map<String, String> mapOfEntityDisplayName = createEntityDisplayNames(entityLog);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID,entityLog, legalBasisMapping, mapOfEntityDisplayName);
FileModel fileModel = FileModel.builder().filename("filename").build();
@ -437,7 +440,7 @@ public class RedactionReportIntegrationTest {
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName", fileModel, excelModel, false);
EntityLog entityLogSecondFile = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<ReportRedactionEntry> reportEntriesSecondFile = entityLogConverterService.convertAndSort(entityLogSecondFile, legalBasisMapping, mapOfEntityDisplayName);
List<ReportRedactionEntry> reportEntriesSecondFile = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID,entityLogSecondFile, legalBasisMapping, mapOfEntityDisplayName);
FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build();
excelTemplateReportGenerationService.generateExcelReport(reportEntriesSecondFile,
placeholders,
@ -464,7 +467,7 @@ public class RedactionReportIntegrationTest {
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
Map<String, String> mapOfEntityDisplayName = createEntityDisplayNames(entityLog);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID,entityLog, legalBasisMapping, mapOfEntityDisplayName);
var imageResource = new ClassPathResource("files/exampleImage.jpg");
FileModel fileModel = FileModel.builder().filename("filename").build();
@ -507,7 +510,7 @@ public class RedactionReportIntegrationTest {
List<FileAttributeConfig> fileAttributeConfigs = objectMapper.readValue(new ClassPathResource("files/scm/fileAttributes.json").getInputStream(), new TypeReference<>() {
});
Map<String, String> mapOfEntityDisplayName = createEntityDisplayNames(entityLog);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, mapOfEntityDisplayName);
when(componentClient.getComponentLog(dossier.getDossierId(), fileModel.getId(), true)).thenReturn(componentLog);
when(fileAttributesConfigClient.getFileAttributeConfigs(dossier.getDossierTemplateId())).thenReturn(fileAttributeConfigs);

16
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterServiceTest.java
View File

@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.report.v1.server.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import java.util.HashMap;
@ -13,6 +11,7 @@ import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
@ -25,11 +24,11 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.redaction.report.v1.server.Application;
import com.iqser.red.service.redaction.report.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierClient;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.tenantcommons.TenantsClient;
@ -41,6 +40,9 @@ import lombok.SneakyThrows;
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public class EntityLogConverterServiceTest {
private static final String DOSSIER_ID = "DossierId";
private static final String FILE_ID = "FileId";
@MockBean
TenantsClient tenantsClient;
@ -53,12 +55,16 @@ public class EntityLogConverterServiceTest {
@MockBean
private ReportStorageService reportStorageService;
@Mock
private ReportTemplateSettings settings;
@InjectMocks
private EntityLogConverterService entityLogConverterService;
@Autowired
private ObjectMapper objectMapper;
@Test
@SneakyThrows
public void testConvertAndSort() {
@ -66,7 +72,9 @@ public class EntityLogConverterServiceTest {
EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, new HashMap<>());
when(settings.getMaxRedactionEntryValueLength()).thenReturn(1000);
when(settings.getMaxSectionLength()).thenReturn(100);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID, entityLog, legalBasisMapping, new HashMap<>());
assertNotNull(reportEntries);
assertFalse(reportEntries.isEmpty());