Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #302: RED-3025: Ported code to ignore hints

Browse Source 
Merge in RED/redaction-service from RED-3025 to master

* commit '010a3ee6ab7f06297c2c37691bcac2f7019fb3a0':
  RED-3025: Ported code to ignore hints
This commit is contained in:
Dominique Eiflaender 2021-12-15 13:26:48 +01:00
commit 1987e7b430
9 changed files with 102 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java
View File

@ -42,6 +42,8 @@ public class Entity implements ReasonHolder {
private boolean isDossierDictionaryEntry;
private boolean ignored;
private Set<Engine> engines = new HashSet<>();
private Set<Entity> references = new HashSet<>();

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java
View File

@ -20,6 +20,7 @@ public class Image implements ReasonHolder {
private int sectionNumber;
private String section;
private int page;
private boolean ignored;
private boolean hasTransparency;
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -156,14 +156,14 @@ public class Section {
@WhenCondition
public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {
return entities.stream().anyMatch(entity -> entity.getType().equals(type));
return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
}
@WhenCondition
public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {
return images.stream().anyMatch(image -> image.getType().equals(type));
return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type));
}

48
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -10,7 +10,9 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer;
@ -82,19 +84,42 @@ public class EntityRedactionService {
surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
}
if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages()
.isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions()
.getImageRecategorization() != null) {
for (Image image : reanalysisSection.getImages()) {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
.getImageRecategorization()) {
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId()
.equals(imageId)) {
image.setType(imageRecategorization.getType());
if (!local && analyzeRequest.getManualRedactions() != null) {
var approvedForceRedactions = analyzeRequest.getManualRedactions().getForceRedactions().stream()
.filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
.filter(fr -> fr.getRequestDate() != null)
.collect(Collectors.toList());
// only approved id removals, that haven't been forced back afterwards
var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
.filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED)
.filter(idr -> idr.getRequestDate() != null)
.filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
.map(IdRemoval::getAnnotationId).collect(Collectors.toSet());
if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
.isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
for (Image image : reanalysisSection.getImages()) {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
.getImageRecategorization()) {
if (imageRecategorization.getStatus()
.equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId()
.equals(imageId)) {
image.setType(imageRecategorization.getType());
}
}
if (idsToRemove.contains(imageId)) {
image.setIgnored(true);
}
}
}
entities.forEach(entity -> entity.getPositionSequences().forEach(ps -> {
if (idsToRemove.contains(ps.getId())) {
entity.setIgnored(true);
}
}));
}
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
@ -149,7 +174,8 @@ public class EntityRedactionService {
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity.getReferences()));
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity
.getReferences()));
}
}
return entitiesPerPage;

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -172,6 +172,7 @@ public class RedactionLogMergeService {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
redactionLogEntry.setHint(false);
redactionLogEntry.setHasBeenRemovedByManualOverride(true);
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
@ -193,7 +194,12 @@ public class RedactionLogMergeService {
var manualForceRedact = (ManualForceRedaction) mrw.getItem();
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(true);
// Forcing a skipped hint should result in a hint
if (dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId)) {
redactionLogEntry.setHint(true);
} else {
redactionLogEntry.setRedacted(true);
}
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");

55
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -50,6 +50,7 @@ import org.springframework.test.context.junit4.SpringRunner;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
@ -628,6 +629,52 @@ public class RedactionIntegrationTest {
}
@Test
@SneakyThrows
public void testIgnoreHint() {
System.out.println("testIgnoreHint");
ClassPathResource pdfFileResource = new ClassPathResource("files/new/test-ignore-hint.pdf");
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var toRemove = IdRemoval.builder()
.annotationId("c630599611e6e3db314518374bcf70f7")
.status(AnnotationStatus.APPROVED)
.user("test")
.removeFromDictionary(false)
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build();
var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(toRemove)).build();
request.setManualRedactions(manualRedactions);
analyzeService.reanalyze(request);
var mergedRedactionLog = redactionController.getRedactionLog(RedactionRequest.builder()
.manualRedactions(manualRedactions)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.build());
var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue();
}
@Test
@Ignore
public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
@ -1469,10 +1516,10 @@ public class RedactionIntegrationTest {
private static String getTemporaryDirectory() {
String tmpdir = System.getProperty("java.io.tmpdir");
if (StringUtils.isNotBlank(tmpdir)) {
return tmpdir;
}
// String tmpdir = System.getProperty("java.io.tmpdir");
// if (StringUtils.isNotBlank(tmpdir)) {
// return tmpdir;
// }
return "/tmp";
}

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt
View File

@ -1660,3 +1660,4 @@ Zyma SA
Zyma SA, Nyon, Switzerland
Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK
Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK
Test Ignored Hint CBI_ADDRESS

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt
View File

@ -85,3 +85,4 @@ Toxicology and Applied Pharmacology
Toxicol Sci
Toxicol Sci.
Toxicol Sci. 1
Test Ignored Hint Published Information

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test-ignore-hint.pdf Normal file
View File

Binary file not shown.