Pull request #500 : RED-5562 34x

Merge in RED/redaction-service from RED-5562-34x to release/3.163.x * commit 'ba17fa77b40df8acb350fd53facfb3413f5af3b7': RED-5562 more symbols RED-5562 more symbols
RED-5562 more symbols
2022-11-15 10:40:36 +01:00 · 2022-11-15 11:28:15 +02:00 · 2022-11-15 11:28:04 +02:00 · 2022-11-15 10:02:44 +01:00 · 2022-11-15 01:38:57 +02:00 · 2022-11-15 00:57:22 +02:00
8 changed files with 198 additions and 113 deletions
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java
@ -1,19 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import lombok.*;
+
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
@Data
@Builder
@NoArgsConstructor
@ -24,6 +19,7 @@ public class Entity implements ReasonHolder {
    private String word;
    private String type;
    private boolean redaction;
+    private boolean falsePositive;
    private String redactionReason;
    private String legalBasis;
    private List<EntityPositionSequence> positionSequences = new ArrayList<>();
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
@ -24,8 +24,11 @@ import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;

@ -87,6 +90,7 @@ public class Section {
        List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
        Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());

        Set<Entity> finalResult = new HashSet<>();

@ -751,6 +755,67 @@ public class Section {
    }


+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactWordPartByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                                      @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                      @Argument(ArgumentType.INTEGER) int group,
+                                      @Argument(ArgumentType.INTEGER) int redactGroup,
+                                      @Argument(ArgumentType.TYPE) String asType,
+                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                      @Argument(ArgumentType.STRING) String reason,
+                                      @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Matcher findMatcher = compiledPattern.matcher(searchText);
+
+        while (findMatcher.find()) {
+            String findMatch = findMatcher.group(group);
+            if (StringUtils.isNotBlank(findMatch)) {
+                Set<Entity> found = findEntities(findMatch.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+
+                for (Entity entity : found) {
+
+                    Matcher redactMatcher = compiledPattern.matcher(entity.getWord());
+
+                    while (redactMatcher.find()) {
+                        String redactMatch = redactMatcher.group(redactGroup);
+                        int start = redactMatcher.start(redactGroup);
+
+                        int i = 0;
+                        List<RedTextPosition> newPositions = new ArrayList<>();
+                        TextPositionSequence newSeq = null;
+
+                        for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
+                            for (TextPositionSequence textPositionSequence : entityPositionSequence.getSequences()) {
+                                for (RedTextPosition textPosition : textPositionSequence.getTextPositions()) {
+                                    if (i >= start && i < start + redactMatch.length()) {
+                                        newPositions.add(textPosition);
+                                        if (newSeq == null) {
+                                            newSeq = textPositionSequence;
+                                        }
+                                    }
+                                    i++;
+                                }
+                            }
+                        }
+
+                        if (newSeq != null) {
+                            newSeq.setTextPositions(newPositions);
+                            entity.setWord(redactMatch);
+                            String plainId = IdBuilder.buildId(List.of(newSeq));
+                            entity.setPositionSequences(List.of(new EntityPositionSequence(plainId, List.of(newSeq), newSeq.getPage())));
+
+                            EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
    @ThenAction
    @SuppressWarnings("unused")
    public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@ -871,6 +936,9 @@ public class Section {
    }


+
+
+
    private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
                                     int ruleNumber, String reason, String legalBasis, Engine engine,
                                     boolean asRecommendation) {
@ -886,13 +954,11 @@ public class Section {
            }
        });

-        return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
    }


-
-
-
    private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {

        entities.forEach(entity -> {
@ -945,22 +1011,25 @@ public class Section {
            singleEntitySet.add(entity);

            EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
+            EntitySearchUtils.removeFalsePositives(singleEntitySet, searchText, dictionary.getType(type), new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY));

-            EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
+            if (!singleEntitySet.isEmpty()) {
+                EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary);

-            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+                EntitySearchUtils.removeEntitiesContainedInLarger(entities);

-            if (addAsRecommendations && !isLocal()) {
-                String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
-                Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
-                Matcher matcher = pattern.matcher(cleanedWord);
+                if (addAsRecommendations && !isLocal()) {
+                    String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
+                    Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
+                    Matcher matcher = pattern.matcher(cleanedWord);

-                while (matcher.find()) {
-                    String match = matcher.group().trim();
-                    if (match.length() >= 3) {
-                        localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
-                        String lastname = match.split(" ")[0];
-                        localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
+                    while (matcher.find()) {
+                        String match = matcher.group().trim();
+                        if (match.length() >= 3) {
+                            localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
+                            String lastname = match.split(" ")[0];
+                            localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
+                        }
                    }
                }
            }
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
@ -1,18 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
@ -22,24 +9,21 @@ import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;

@Slf4j
@Service
@ -51,7 +35,6 @@ public class EntityRedactionService {
    private final SurroundingWordsService surroundingWordsService;


-
    public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer,
                                     AnalyzeRequest analyzeRequest, NerEntities nerEntities) {

@ -262,7 +245,8 @@ public class EntityRedactionService {
            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
        }

-        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions), nerFound);
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
    }


--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
@ -1,28 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

+import java.util.*;
+import java.util.stream.Collectors;
+
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
@ -33,18 +20,23 @@ public class EntitySearchUtils {
        return searchImplementation.atLeastOneMatches(sectionText);
    }

+
+    public void removeFalsePositives(Set<Entity> found, String inputString, DictionaryModel type, FindEntityDetails details){
+        Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
+        markFalsePositives(found, falsePositives);
+        found.removeIf(f -> f.isFalsePositive());
+    }
+
    public Set<Entity> findEntities(String inputString, SearchImplementation searchImplementation, DictionaryModel type, FindEntityDetails details) {

        Set<Entity> found = find(inputString, searchImplementation, details);

        if (details.getEntityType() == EntityType.RECOMMENDATION) {
            Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
-            removeFalsePositives(found, falseRecommendations);
-//            found.addAll(falseRecommendations);
+            markFalsePositives(found, falseRecommendations);
        } else {
            Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
-            removeFalsePositives(found, falsePositives);
-//            found.addAll(falsePositives);
+            markFalsePositives(found, falsePositives);
        }

        return found;
@ -93,7 +85,7 @@ public class EntitySearchUtils {
            }
        }

-        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()){
+        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()) {
            applyResizeRedactions(entities, manualRedactions);
        }

@ -131,7 +123,11 @@ public class EntitySearchUtils {
    }


-    public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
+    public void markFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
+
+        if (entities.size() == 2) {
+            log.info("asd");
+        }

        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : falsePositives) {
@ -142,8 +138,12 @@ public class EntitySearchUtils {
                }
            }
        }
-        wordsToRemove.forEach(entities::remove);
-        entities.removeAll(falsePositives);
+        wordsToRemove.forEach(e -> e.setFalsePositive(true));
+        for (var entity : entities) {
+            if (falsePositives.contains(entity)) {
+                entity.setFalsePositive(true);
+            }
+        }
    }


@ -165,9 +165,9 @@ public class EntitySearchUtils {
                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                        wordsToRemove.add(word);
-                    } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
+                    } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
                            inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
-                        if(inner.isResized()){
+                        if (inner.isResized()) {
                            wordsToRemove.add(word);
                        } else {
                            wordsToRemove.add(inner);
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java
@ -1,15 +1,26 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import lombok.extern.slf4j.Slf4j;
+
 import java.util.Set;
 import java.util.regex.Pattern;

+@Slf4j
 public class SeparatorUtils {

+    private final static Pattern punctuationPattern = Pattern.compile("\\p{Punct}");
    private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E'
            , '\u2039', '\u203A');

+    private final static Set<Integer> japaneseAltPunctuationMarks = Set.of(65288, 65289, 65294, 65339, 65341, 65371, 65373, 65375, 65376, 12443, 12444, 65309, 65306);

    public static boolean isSeparator(char c) {
-        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || quotes.contains(c);
+        return Character.isWhitespace(c) || punctuationPattern.matcher(String.valueOf(c)).matches() || quotes.contains(c) || isJapaneseSeparator(c);
    }
+
+    public static boolean isJapaneseSeparator(char c) {
+        var intValue = (int) c;
+        return intValue >= 12288 && intValue <= 12336 || japaneseAltPunctuationMarks.contains(intValue);
+    }
+
 }
--- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java
@ -5,9 +5,11 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.MessageType;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

 import java.time.OffsetDateTime;
@ -19,6 +21,9 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest

    public static final String FILE_NAME = "test-file";

+    @Autowired
+    private AnnotationService annotationService;
+
    @Test
    @SneakyThrows
    public void testFile() {
@ -81,7 +86,19 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
        redactionMessageReceiver.receiveAnalyzeRequest(in, false);


-        // var log = redactionStorageService.getRedactionLog("dossierId","fileId");
+//        var redactionLog = redactionStorageService.getRedactionLog("dossierId","fileId");
+//        redactionLog.getRedactionLogEntry().forEach(entry ->{
+//            log.info("Entry {}", entry);
+//        });
+//
+//        var annotated = annotationService.annotate(AnnotateRequest.builder().fileId("fileId").dossierId("dossierId").build());
+//        annotated.getDocument();
+//
+//        File tempFile = File.createTempFile("annotated",".pdf");
+//        IOUtils.write(annotated.getDocument(), new FileOutputStream(tempFile));
+//        log.warn("File saved to: {}",tempFile.getAbsolutePath());
+//        Runtime.getRuntime().exec("open "+tempFile.getAbsolutePath());
    }

+
 }
--- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
@ -1,17 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.realdata;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.nullable;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.*;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
@ -30,27 +34,14 @@ import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
 import org.springframework.test.context.junit4.SpringRunner;

-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
-import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;

-import lombok.SneakyThrows;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -143,6 +134,7 @@ public class LiveDataIntegrationTest {
            if (found.isPresent()) {
                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
                type.setEntries(getEntries(typeName, type.getTypeId()));
+                type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId()));

                return type;
            } else {
@ -196,7 +188,22 @@ public class LiveDataIntegrationTest {

        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
        for (var resource : dictionaryResources) {
-            if (Objects.requireNonNull(resource.getFilename()).contains(typeName)) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) {
+
+                List<String> lines = IOUtils.readLines(resource.getInputStream());
+                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
+
+            }
+        }
+        return new ArrayList<>();
+    }
+
+    @SneakyThrows
+    private List<DictionaryEntry> getFalsePositiveEntries(String typeName, String typeId) {
+
+        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
+        for (var resource : dictionaryResources) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) {

                List<String> lines = IOUtils.readLines(resource.getInputStream());
                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
--- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/dictionaries/CBI_author_false_positive.txt
+++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/dictionaries/CBI_author_false_positive.txt
@ -0,0 +1 @@
+Salt), CGA376944 (S-enantiomer
Author	SHA1	Message	Date
Timo Bejan	884648f386	Pull request #500 : RED-5562 34x Merge in RED/redaction-service from RED-5562-34x to release/3.163.x * commit 'ba17fa77b40df8acb350fd53facfb3413f5af3b7': RED-5562 more symbols RED-5562 more symbols	2022-11-15 10:40:36 +01:00
Timo Bejan	ba17fa77b4	RED-5562 more symbols	2022-11-15 11:28:15 +02:00
Timo Bejan	a8113ba7f5	RED-5562 more symbols	2022-11-15 11:28:04 +02:00
Timo Bejan	35f7b1ce1f	Pull request #499 : RED-5562 japanse space characters Merge in RED/redaction-service from RED-5562-34x to release/3.163.x * commit '5d68955be68a6a5bf0d1690c2f69419f78a4291f': RED-5562 improved pattern compile RED-5562 japanse space characters - fixed PMD RED-5562 japanse space characters	2022-11-15 10:02:44 +01:00
Timo Bejan	5d68955be6	RED-5562 improved pattern compile	2022-11-15 01:38:57 +02:00
Timo Bejan	37edffc75f	RED-5562 japanse space characters - fixed PMD	2022-11-15 00:57:22 +02:00
Timo Bejan	7e95174f57	RED-5562 japanse space characters	2022-11-14 23:33:42 +02:00
Dominique Eiflaender	c304ed79a0	Pull request #497 : RED-5459: Backported rule redactWordPartByRegEx Merge in RED/redaction-service from RED-5459 to release/3.163.x * commit '8f3afa962b0afcd5d7e4f42ba80c37e743596b36': RED-5459: Backported rule redactWordPartByRegEx	2022-11-04 15:55:34 +01:00
deiflaender	8f3afa962b	RED-5459: Backported rule redactWordPartByRegEx	2022-11-04 15:36:38 +01:00
Kresnadi Budisantoso	f42537ea1a	Pull request #479 : RED-5139: Bugfix if false positives are added, for rule added entries Merge in RED/redaction-service from kbudisantoso/RED-5139-backport to release/3.163.x * commit '90b7875c0b00298cb6972ee2fed81cad63dfcf08': RED-5139: Bugfix if false positives are added, for rule added entries	2022-09-28 09:24:30 +02:00
deiflaender	90b7875c0b	RED-5139: Bugfix if false positives are added, for rule added entries	2022-09-27 17:51:12 +02:00
Timo Bejan	de841452ba	checkstyle fix	2022-08-19 17:31:50 +03:00
Timo Bejan	96e8485afe	RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive	2022-08-19 17:29:43 +03:00