Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-5139: Bugfix if false positives are added, for rule added entries

Browse Source
This commit is contained in:
Philipp Schramm 2022-09-26 10:09:55 +02:00
parent 142f5256ae
commit 77f298e1dc
4 changed files with 104 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -1002,7 +1002,6 @@ public class Section {
} }
} }
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
int ruleNumber, String reason, String legalBasis, Engine engine, int ruleNumber, String reason, String legalBasis, Engine engine,
boolean asRecommendation) { boolean asRecommendation) {
@ -1019,7 +1018,9 @@ public class Section {
}); });
var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions); var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet()); return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
} }
@ -1060,25 +1061,18 @@ public class Section {
} else { } else {
String word = value.toString(); String word = value.toString();
Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY); Set<Entity> found = findEntities(word.trim(), type, false, redact, ruleNumber, reason, legalBasis, Engine.RULE, false);
entity.setRedaction(redact);
entity.setMatchedRule(ruleNumber);
entity.setRedactionReason(reason);
entity.setTargetSequences(value.getTextBlocks()
.stream()
.map(TextBlock::getSequences)
.flatMap(Collection::stream)
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
entity.setLegalBasis(legalBasis);
Set<Entity> singleEntitySet = new HashSet<>(); for (Entity entity: found) {// Should be found
singleEntitySet.add(entity); entity.setTargetSequences(value.getTextBlocks()
.stream()
.map(TextBlock::getSequences)
.flatMap(Collection::stream)
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions); }
EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary); EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
if (addAsRecommendations && !isLocal()) { if (addAsRecommendations && !isLocal()) {
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " "; String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -1,15 +1,29 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils; package com.iqser.red.service.redaction.v1.server.redaction.utils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j @Slf4j
@UtilityClass @UtilityClass
@SuppressWarnings("PMD") @SuppressWarnings("PMD")
@ -238,6 +252,8 @@ public class EntitySearchUtils {
existing.setLegalBasis(found.getLegalBasis()); existing.setLegalBasis(found.getLegalBasis());
existing.setMatchedRule(found.getMatchedRule()); existing.setMatchedRule(found.getMatchedRule());
existing.setRedactionReason(found.getRedactionReason()); existing.setRedactionReason(found.getRedactionReason());
existing.setFalsePositive(found.isFalsePositive());
if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
|| existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) { || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
existing.setEntityType(EntityType.ENTITY); existing.setEntityType(EntityType.ENTITY);

113
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,36 +1,30 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import com.amazonaws.services.s3.AmazonS3; import static org.assertj.core.api.Assertions.assertThat;
import com.fasterxml.jackson.core.type.TypeReference; import static org.mockito.Mockito.when;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import java.io.BufferedReader;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import java.io.ByteArrayInputStream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import java.io.File;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; import java.io.FileInputStream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*; import java.io.FileOutputStream;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import java.io.IOException;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; import java.io.InputStream;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import java.io.InputStreamReader;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; import java.net.URL;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import java.nio.charset.StandardCharsets;
import com.iqser.red.service.redaction.v1.model.*; import java.time.OffsetDateTime;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import java.time.ZoneOffset;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import java.util.ArrayList;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; import java.util.Collections;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import java.util.HashMap;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import java.util.List;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import java.util.Map;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; import java.util.UUID;
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; import java.util.stream.Collectors;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore; import org.junit.Ignore;
@ -55,16 +49,49 @@ import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import java.io.*; import com.amazonaws.services.s3.AmazonS3;
import java.net.URL; import com.fasterxml.jackson.core.type.TypeReference;
import java.nio.charset.StandardCharsets; import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.OffsetDateTime; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import java.time.ZoneOffset; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import java.util.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import java.util.stream.Collectors; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import static org.assertj.core.api.Assertions.assertThat; import lombok.SneakyThrows;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -1562,6 +1589,12 @@ public class RedactionIntegrationTest {
.map(this::cleanDictionaryEntry) .map(this::cleanDictionaryEntry)
.collect(Collectors.toSet())); .collect(Collectors.toSet()));
falsePositive.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_author_false_positive.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
} }
private void loadOnlyDictionaryForSimpleFile() { private void loadOnlyDictionaryForSimpleFile() {

0
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author_false_positive.txt Normal file
View File