Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-3674: Fixed force redactions overrides idRemoval with different id

Browse Source
This commit is contained in:
aoezyetimoglu 2022-05-23 17:38:23 +02:00
parent 4aaef260d7
commit 87d64b6d12
8 changed files with 315 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -82,9 +82,10 @@ public class EntityRedactionService {
var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
.filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
.filter(idr -> idr.getRequestDate() != null)
.filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
.filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
.map(IdRemoval::getAnnotationId).collect(Collectors.toSet());
if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
.isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
for (Image image : reanalysisSection.getImages()) {

68
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
@ -58,6 +59,7 @@ import java.io.*;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.*;
import java.util.stream.Collectors;
@ -715,6 +717,72 @@ public class RedactionIntegrationTest {
}
@Test
public void testRemovePublishedInformations() throws IOException {
long start = System.currentTimeMillis();
ClassPathResource colorsResource = new ClassPathResource("colors/colors.json");
var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
ClassPathResource typeResource = new ClassPathResource("colors/types.json");
TypeReference<List<Type>> typeRefForTypes = new TypeReference<>(){};
List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getIdsToRemove().add(IdRemoval.builder()
.annotationId("308dab9015bfafd911568cffe0a7f7de")
.fileId(TEST_FILE_ID)
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.of(2022,05,23,8,30,07,475479, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022,05,23,8,30,07,483651, ZoneOffset.UTC))
.build());
manualRedactions.getForceRedactions().add(ManualForceRedaction.builder()
.annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID)
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.of(2022,05,23,9,30,15,4653, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022,05,23,9,30,15,794, ZoneOffset.UTC))
.build());
manualRedactions.getIdsToRemove().add(IdRemoval.builder()
.annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID)
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.of(2022,05,23,8,30,23,961721, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022,05,23,8,30,23,96528, ZoneOffset.UTC))
.build());
request.setManualRedactions(manualRedactions);
AnalyzeResult result = analyzeService.analyze(request);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
.manualRedactions(manualRedactions)
.colors(colors)
.types(types)
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
}
long end = System.currentTimeMillis();
System.out.println("duration: " + (end - start));
System.out.println("numberOfPages: " + result.getNumberOfPages());
}
@Test
public void testTableRedaction() throws IOException {

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java
View File

@ -1,6 +1,10 @@
package com.iqser.red.service.redaction.v1.server.annotate;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import lombok.AllArgsConstructor;
import lombok.Builder;
@ -17,4 +21,6 @@ public class AnnotateRequest {
private String dossierTemplateId;
private String fileId;
private ManualRedactions manualRedactions;
private Colors colors;
private List<Type> types;
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java
View File

@ -46,6 +46,8 @@ public class AnnotationService {
.manualRedactions(annotateRequest.getManualRedactions())
.dossierId(annotateRequest.getDossierId())
.dossierTemplateId(annotateRequest.getDossierTemplateId())
.colors(annotateRequest.getColors())
.types(annotateRequest.getTypes())
.build());
var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/colors.json Normal file
View File

@ -0,0 +1,13 @@
{
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"defaultColor": "#9398a0",
"requestAdd": "#04b093",
"requestRemove": "#04b093",
"notRedacted": "#c498fa",
"analysisColor": "#dd4d50",
"updatedColor": "#fdbd00",
"dictionaryRequestColor": "#5b97db",
"manualRedactionColor": "#9398a0",
"previewColor": "#9398a0",
"ignoredHintColor": "#e7d4ff"
}

223
redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/types.json Normal file
View File

@ -0,0 +1,223 @@
[
{
"id": "CBI_address:31039447-9040-4376-9ca7-614e56b284b9",
"type": "CBI_address",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"rank": 140,
"isHint": false,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": false,
"isRecommendation": false,
"description": "All site names and addresses, and location (e.g. Syngenta, Monthey, GPS Co-ordinates, Mr Smith of … providing the…). Except addresses in published literature and the applicant address.",
"addToDictionaryAction": true,
"label": "CBI Address",
"hasDictionary": true,
"systemManaged": false,
"autoHideSkipped": false
},
{
"id": "CBI_author:31039447-9040-4376-9ca7-614e56b284b9",
"type": "CBI_author",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"rank": 130,
"isHint": false,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": false,
"isRecommendation": false,
"description": "All authors named in the study documentation. Except names in published literature.",
"addToDictionaryAction": true,
"label": "CBI Author",
"hasDictionary": true,
"systemManaged": false,
"autoHideSkipped": false
},
{
"id": "PII:31039447-9040-4376-9ca7-614e56b284b9",
"type": "PII",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"rank": 150,
"isHint": false,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": false,
"isRecommendation": false,
"description": "Not authors but listed in the document: Names, signatures, telephone, email etc.; e.g. Reg Manager, QA Manager",
"addToDictionaryAction": true,
"label": "PII",
"hasDictionary": true,
"systemManaged": false,
"autoHideSkipped": false
},
{
"id": "formula:31039447-9040-4376-9ca7-614e56b284b9",
"type": "formula",
"hexColor": "#036ffc",
"recommendationHexColor": "#8df06c",
"rank": 1002,
"isHint": true,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": true,
"isRecommendation": false,
"description": "Empty dictionary used to configure formula colors.",
"addToDictionaryAction": false,
"label": "Formula",
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false
},
{
"id": "isHint_only:31039447-9040-4376-9ca7-614e56b284b9",
"type": "isHint_only",
"hexColor": "#fa98f7",
"recommendationHexColor": "#8df06c",
"rank": 50,
"isHint": true,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": true,
"isRecommendation": false,
"description": "Entries of this dictionary will be highlighted only",
"addToDictionaryAction": false,
"label": "isHint Only",
"hasDictionary": true,
"systemManaged": false,
"autoHideSkipped": false
},
{
"id": "image:31039447-9040-4376-9ca7-614e56b284b9",
"type": "image",
"hexColor": "#bdd6ff",
"recommendationHexColor": "#8df06c",
"rank": 999,
"isHint": true,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": true,
"isRecommendation": false,
"description": "Empty dictionary used to configure image colors.",
"addToDictionaryAction": false,
"label": "Image",
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false
},
{
"id": "logo:31039447-9040-4376-9ca7-614e56b284b9",
"type": "logo",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"rank": 1001,
"isHint": false,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": true,
"isRecommendation": false,
"description": "Empty dictionary used to configure logo colors.",
"addToDictionaryAction": false,
"label": "Logo",
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false
},
{
"id": "must_redact:31039447-9040-4376-9ca7-614e56b284b9",
"type": "must_redact",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"rank": 100,
"isHint": false,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": true,
"isRecommendation": false,
"description": "Entries of this dictionary get redacted wherever found.",
"addToDictionaryAction": false,
"label": "Must Redact",
"hasDictionary": true,
"systemManaged": false,
"autoHideSkipped": false
},
{
"id": "ocr:31039447-9040-4376-9ca7-614e56b284b9",
"type": "ocr",
"hexColor": "#bdd6ff",
"recommendationHexColor": "#8df06c",
"rank": 1000,
"isHint": true,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": true,
"isRecommendation": false,
"description": "Empty dictionary used to configure ocr colors.",
"addToDictionaryAction": false,
"label": "Ocr",
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false
},
{
"id": "signature:31039447-9040-4376-9ca7-614e56b284b9",
"type": "signature",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"rank": 1003,
"isHint": false,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": true,
"isRecommendation": false,
"description": "Empty dictionary used to configure signature colors.",
"addToDictionaryAction": false,
"label": "Signature",
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false
},
{
"id": "imported_redaction:31039447-9040-4376-9ca7-614e56b284b9",
"type": "imported_redaction",
"hexColor": "#f0f0c0",
"recommendationHexColor": "#8df06c",
"rank": 9999,
"isHint": false,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": false,
"isRecommendation": false,
"description": "Redaction Annotations that were imported from documents",
"addToDictionaryAction": false,
"label": "Imported Redaction",
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": true
},
{
"id": "published_information:31039447-9040-4376-9ca7-614e56b284b9",
"type": "published_information",
"hexColor": "#85ebff",
"recommendationHexColor": "#8df06c",
"rank": 70,
"isHint": true,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": false,
"isRecommendation": false,
"description": "Manual managed list of public journals and papers that need no redaction",
"addToDictionaryAction": true,
"label": "Published Information",
"hasDictionary": true,
"systemManaged": false,
"autoHideSkipped": false
},
{
"id": "dossier_redaction:31039447-9040-4376-9ca7-614e56b284b9:5dfb2724-74a4-4a1a-a1eb-165e7943ffcd",
"type": "dossier_redaction",
"hexColor": "#9398a0",
"recommendationHexColor": null,
"rank": 1500,
"isHint": false,
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
"isCaseInsensitive": false,
"isRecommendation": false,
"description": "Entries in this dictionary will only be redacted in this dossier",
"addToDictionaryAction": false,
"label": "Dossier Redaction",
"hasDictionary": true,
"systemManaged": true,
"autoHideSkipped": false
}
]

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt
View File

@ -86,3 +86,4 @@ Toxicol Sci
Toxicol Sci.
Toxicol Sci. 1
Test Ignored Hint Published Information
Workshop

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/PublishedInformationTest.pdf Normal file
View File

Binary file not shown.