diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 779d8217..e7d2ce07 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1002,7 +1002,6 @@ public class Section { } } - private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine, boolean asRecommendation) { @@ -1019,7 +1018,9 @@ public class Section { }); var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions); + return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet()); + } @@ -1060,25 +1061,18 @@ public class Section { } else { String word = value.toString(); - Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY); - entity.setRedaction(redact); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - entity.setTargetSequences(value.getTextBlocks() - .stream() - .map(TextBlock::getSequences) - .flatMap(Collection::stream) - .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted - entity.setLegalBasis(legalBasis); + Set found = findEntities(word.trim(), type, false, redact, ruleNumber, reason, legalBasis, Engine.RULE, false); - Set singleEntitySet = new HashSet<>(); - singleEntitySet.add(entity); + for (Entity entity: found) {// Should be found + entity.setTargetSequences(value.getTextBlocks() + .stream() + .map(TextBlock::getSequences) + .flatMap(Collection::stream) + .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted - EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions); + } - EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary); - - EntitySearchUtils.removeEntitiesContainedInLarger(entities); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); if (addAsRecommendations && !isLocal()) { String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " "; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 80c7d57c..ea4287ce 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -1,15 +1,29 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.Image; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; + import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; -import java.util.*; -import java.util.stream.Collectors; - @Slf4j @UtilityClass @SuppressWarnings("PMD") @@ -238,6 +252,8 @@ public class EntitySearchUtils { existing.setLegalBasis(found.getLegalBasis()); existing.setMatchedRule(found.getMatchedRule()); existing.setRedactionReason(found.getRedactionReason()); + existing.setFalsePositive(found.isFalsePositive()); + if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) { existing.setEntityType(EntityType.ENTITY); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index fe31be8b..cd2556ac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1,36 +1,30 @@ package com.iqser.red.service.redaction.v1.server; -import com.amazonaws.services.s3.AmazonS3; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*; -import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; -import com.iqser.red.service.redaction.v1.model.*; -import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; -import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; -import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; -import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; -import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; -import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; -import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.controller.RedactionController; -import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; -import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; -import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; -import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; -import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; -import lombok.SneakyThrows; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -55,16 +49,49 @@ import org.springframework.context.annotation.Primary; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit4.SpringRunner; -import java.io.*; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.util.*; -import java.util.stream.Collectors; +import com.amazonaws.services.s3.AmazonS3; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; +import com.iqser.red.service.redaction.v1.model.AnalyzeResult; +import com.iqser.red.service.redaction.v1.model.FileAttribute; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.RedactionRequest; +import com.iqser.red.service.redaction.v1.model.RedactionResult; +import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; +import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; +import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.controller.RedactionController; +import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; +import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; +import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; +import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; +import lombok.SneakyThrows; @RunWith(SpringRunner.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -1562,6 +1589,12 @@ public class RedactionIntegrationTest { .map(this::cleanDictionaryEntry) .collect(Collectors.toSet())); + falsePositive.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/CBI_author_false_positive.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); + } private void loadOnlyDictionaryForSimpleFile() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author_false_positive.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author_false_positive.txt new file mode 100644 index 00000000..e69de29b