Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-5139: Bugfix if false positives are added, for rule added entries

Browse Source
This commit is contained in:
Philipp Schramm 2022-09-26 13:35:31 +02:00
parent 0c95bf3bbc
commit 69ea9a6f93
3 changed files with 73 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -1072,7 +1072,6 @@ public class Section {
} }
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
if (addAsRecommendations && !isLocal()) { if (addAsRecommendations && !isLocal()) {

113
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,36 +1,30 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import com.amazonaws.services.s3.AmazonS3; import static org.assertj.core.api.Assertions.assertThat;
import com.fasterxml.jackson.core.type.TypeReference; import static org.mockito.Mockito.when;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import java.io.BufferedReader;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import java.io.ByteArrayInputStream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import java.io.File;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; import java.io.FileInputStream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*; import java.io.FileOutputStream;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import java.io.IOException;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; import java.io.InputStream;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import java.io.InputStreamReader;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; import java.net.URL;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import java.nio.charset.StandardCharsets;
import com.iqser.red.service.redaction.v1.model.*; import java.time.OffsetDateTime;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import java.time.ZoneOffset;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import java.util.ArrayList;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; import java.util.Collections;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import java.util.HashMap;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import java.util.List;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import java.util.Map;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; import java.util.UUID;
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; import java.util.stream.Collectors;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore; import org.junit.Ignore;
@ -55,16 +49,49 @@ import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import java.io.*; import com.amazonaws.services.s3.AmazonS3;
import java.net.URL; import com.fasterxml.jackson.core.type.TypeReference;
import java.nio.charset.StandardCharsets; import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.OffsetDateTime; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import java.time.ZoneOffset; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import java.util.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import java.util.stream.Collectors; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import static org.assertj.core.api.Assertions.assertThat; import lombok.SneakyThrows;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -1562,6 +1589,12 @@ public class RedactionIntegrationTest {
.map(this::cleanDictionaryEntry) .map(this::cleanDictionaryEntry)
.collect(Collectors.toSet())); .collect(Collectors.toSet()));
falsePositive.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_author_false_positive.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
} }
private void loadOnlyDictionaryForSimpleFile() { private void loadOnlyDictionaryForSimpleFile() {

0
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author_false_positive.txt Normal file
View File