RED-4510: Added generated RedactionLog Json files and optimizations for RulesTest.java for Bamboo

redaction-service-v1/pom.xml

@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.20.0</version>
+                <version>1.17.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.254.0</persistence-service.version>
+        <persistence-service.version>1.240.0</persistence-service.version>
     </properties>
 
     <dependencies>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,14 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import lombok.*;
-
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
 @Data
 @Builder
 @NoArgsConstructor
@@ -19,7 +24,6 @@ public class Entity implements ReasonHolder {
     private String word;
     private String type;
     private boolean redaction;
-    private boolean falsePositive;
     private String redactionReason;
     private String legalBasis;
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -24,11 +24,8 @@ import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 
@@ -90,7 +87,6 @@ public class Section {
         List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
         Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
         EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
 
         Set<Entity> finalResult = new HashSet<>();
 
@@ -755,67 +751,6 @@ public class Section {
     }
 
 
-    @ThenAction
-    @SuppressWarnings("unused")
-    public void redactWordPartByRegEx(@Argument(ArgumentType.REGEX) String pattern,
-                                      @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                                      @Argument(ArgumentType.INTEGER) int group,
-                                      @Argument(ArgumentType.INTEGER) int redactGroup,
-                                      @Argument(ArgumentType.TYPE) String asType,
-                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                      @Argument(ArgumentType.STRING) String reason,
-                                      @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
-
-        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
-
-        Matcher findMatcher = compiledPattern.matcher(searchText);
-
-        while (findMatcher.find()) {
-            String findMatch = findMatcher.group(group);
-            if (StringUtils.isNotBlank(findMatch)) {
-                Set<Entity> found = findEntities(findMatch.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
-
-                for (Entity entity : found) {
-
-                    Matcher redactMatcher = compiledPattern.matcher(entity.getWord());
-
-                    while (redactMatcher.find()) {
-                        String redactMatch = redactMatcher.group(redactGroup);
-                        int start = redactMatcher.start(redactGroup);
-
-                        int i = 0;
-                        List<RedTextPosition> newPositions = new ArrayList<>();
-                        TextPositionSequence newSeq = null;
-
-                        for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
-                            for (TextPositionSequence textPositionSequence : entityPositionSequence.getSequences()) {
-                                for (RedTextPosition textPosition : textPositionSequence.getTextPositions()) {
-                                    if (i >= start && i < start + redactMatch.length()) {
-                                        newPositions.add(textPosition);
-                                        if (newSeq == null) {
-                                            newSeq = textPositionSequence;
-                                        }
-                                    }
-                                    i++;
-                                }
-                            }
-                        }
-
-                        if (newSeq != null) {
-                            newSeq.setTextPositions(newPositions);
-                            entity.setWord(redactMatch);
-                            String plainId = IdBuilder.buildId(List.of(newSeq));
-                            entity.setPositionSequences(List.of(new EntityPositionSequence(plainId, List.of(newSeq), newSeq.getPage())));
-
-                            EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-
     @ThenAction
     @SuppressWarnings("unused")
     public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@@ -936,9 +871,6 @@ public class Section {
     }
 
 
-
-
-
     private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
                                      int ruleNumber, String reason, String legalBasis, Engine engine,
                                      boolean asRecommendation) {
@@ -954,11 +886,13 @@ public class Section {
             }
         });
 
-        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
+        return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
     }
 
 
+
+
+
     private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {
 
         entities.forEach(entity -> {
@@ -1011,25 +945,22 @@ public class Section {
             singleEntitySet.add(entity);
 
             EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
-            EntitySearchUtils.removeFalsePositives(singleEntitySet, searchText, dictionary.getType(type), new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY));
 
-            if (!singleEntitySet.isEmpty()) {
-                EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary);
+            EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
 
-                EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
 
-                if (addAsRecommendations && !isLocal()) {
-                    String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
-                    Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
-                    Matcher matcher = pattern.matcher(cleanedWord);
+            if (addAsRecommendations && !isLocal()) {
+                String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
+                Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
+                Matcher matcher = pattern.matcher(cleanedWord);
 
-                    while (matcher.find()) {
-                        String match = matcher.group().trim();
-                        if (match.length() >= 3) {
-                            localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
-                            String lastname = match.split(" ")[0];
-                            localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
-                        }
+                while (matcher.find()) {
+                    String match = matcher.group().trim();
+                    if (match.length() >= 3) {
+                        localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
+                        String lastname = match.split(" ")[0];
+                        localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
                     }
                 }
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -1,5 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
@@ -9,21 +22,24 @@ import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 @Slf4j
 @Service
@@ -35,6 +51,7 @@ public class EntityRedactionService {
     private final SurroundingWordsService surroundingWordsService;
 
 
+
     public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer,
                                      AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
 
@@ -245,8 +262,7 @@ public class EntityRedactionService {
             nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
         }
 
-        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
+        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions), nerFound);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,15 +1,28 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 @UtilityClass
 @SuppressWarnings("PMD")
@@ -20,23 +33,18 @@ public class EntitySearchUtils {
         return searchImplementation.atLeastOneMatches(sectionText);
     }
 
-
-    public void removeFalsePositives(Set<Entity> found, String inputString, DictionaryModel type, FindEntityDetails details){
-        Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
-        markFalsePositives(found, falsePositives);
-        found.removeIf(f -> f.isFalsePositive());
-    }
-
     public Set<Entity> findEntities(String inputString, SearchImplementation searchImplementation, DictionaryModel type, FindEntityDetails details) {
 
         Set<Entity> found = find(inputString, searchImplementation, details);
 
         if (details.getEntityType() == EntityType.RECOMMENDATION) {
             Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
-            markFalsePositives(found, falseRecommendations);
+            removeFalsePositives(found, falseRecommendations);
+            found.addAll(falseRecommendations);
         } else {
             Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
-            markFalsePositives(found, falsePositives);
+            removeFalsePositives(found, falsePositives);
+            found.addAll(falsePositives);
         }
 
         return found;
@@ -85,7 +93,7 @@ public class EntitySearchUtils {
             }
         }
 
-        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()) {
+        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()){
             applyResizeRedactions(entities, manualRedactions);
         }
 
@@ -123,11 +131,7 @@ public class EntitySearchUtils {
     }
 
 
-    public void markFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
-
-        if (entities.size() == 2) {
-            log.info("asd");
-        }
+    public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
 
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : falsePositives) {
@@ -138,12 +142,8 @@ public class EntitySearchUtils {
                 }
             }
         }
-        wordsToRemove.forEach(e -> e.setFalsePositive(true));
-        for (var entity : entities) {
-            if (falsePositives.contains(entity)) {
-                entity.setFalsePositive(true);
-            }
-        }
+        wordsToRemove.forEach(entities::remove);
+        entities.removeAll(falsePositives);
     }
 
 
@@ -152,22 +152,13 @@ public class EntitySearchUtils {
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
-//                // skip cross-type false positives
-//                if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){
-//                    continue;
-//                }
-//                // skip cross-type false reccommendations
-//                if(word.getEntityType() == EntityType.FALSE_RECOMMENDATION && inner.getEntityType() == EntityType.RECOMMENDATION && !inner.getType().equals(word.getType())){
-//                    continue;
-//                }
-
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                     if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                         wordsToRemove.add(word);
-                    } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
+                    } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
                             inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
-                        if (inner.isResized()) {
+                        if(inner.isResized()){
                             wordsToRemove.add(word);
                         } else {
                             wordsToRemove.add(inner);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java

@@ -1,26 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import lombok.extern.slf4j.Slf4j;
-
 import java.util.Set;
 import java.util.regex.Pattern;
 
-@Slf4j
 public class SeparatorUtils {
 
-    private final static Pattern punctuationPattern = Pattern.compile("\\p{Punct}");
     private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E'
             , '\u2039', '\u203A');
 
-    private final static Set<Integer> japaneseAltPunctuationMarks = Set.of(65288, 65289, 65294, 65339, 65341, 65371, 65373, 65375, 65376, 12443, 12444, 65309, 65306);
 
     public static boolean isSeparator(char c) {
-        return Character.isWhitespace(c) || punctuationPattern.matcher(String.valueOf(c)).matches() || quotes.contains(c) || isJapaneseSeparator(c);
+        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || quotes.contains(c);
     }
-
-    public static boolean isJapaneseSeparator(char c) {
-        var intValue = (int) c;
-        return intValue >= 12288 && intValue <= 12336 || japaneseAltPunctuationMarks.contains(intValue);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 import lombok.SneakyThrows;
@@ -63,8 +62,11 @@ public class FileSystemBackedStorageService implements StorageService {
     }
 
     private ObjectMapper getMapper() {
-
-        return ObjectMapperFactory.create();
+        var objectMapper = new ObjectMapper();
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        objectMapper.registerModule(new JavaTimeModule());
+        objectMapper.findAndRegisterModules();
+        return objectMapper;
     }
 
     @Override

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -113,12 +113,15 @@ public class RulesTest {
     private static final String IMPORTED_REDACTION = "imported_redaction";
     private static final String PII = "PII";
     private static final String RESOURCES_PATH = "src/test/resources/";
+
     private static final String REDACTION_LOG_PATH = "RedactionLog/";
     private static final String REDACTION_LOG_RESOURCES_PATH = RESOURCES_PATH + REDACTION_LOG_PATH;
     private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
     private final static String TEST_DOSSIER_ID = "123";
     private String TEST_FILE_ID = "123";
+
     private int counter = 0;
+
     private int fileSize = 0;
     private final Map<String, List<String>> dictionary = new HashMap<>();
     private final Map<String, List<String>> dossierDictionary = new HashMap<>();
@@ -313,7 +316,7 @@ public class RulesTest {
         assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
         assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
 
-        assertThat(redactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size());
+        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
         assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
 
         for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java

@@ -5,11 +5,9 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.MessageType;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
 import java.time.OffsetDateTime;
@@ -21,9 +19,6 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
 
     public static final String FILE_NAME = "test-file";
 
-    @Autowired
-    private AnnotationService annotationService;
-
     @Test
     @SneakyThrows
     public void testFile() {
@@ -86,19 +81,7 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
         redactionMessageReceiver.receiveAnalyzeRequest(in, false);
 
 
-//        var redactionLog = redactionStorageService.getRedactionLog("dossierId","fileId");
-//        redactionLog.getRedactionLogEntry().forEach(entry ->{
-//            log.info("Entry {}", entry);
-//        });
-//
-//        var annotated = annotationService.annotate(AnnotateRequest.builder().fileId("fileId").dossierId("dossierId").build());
-//        annotated.getDocument();
-//
-//        File tempFile = File.createTempFile("annotated",".pdf");
-//        IOUtils.write(annotated.getDocument(), new FileOutputStream(tempFile));
-//        log.warn("File saved to: {}",tempFile.getAbsolutePath());
-//        Runtime.getRuntime().exec("open "+tempFile.getAbsolutePath());
+        // var log = redactionStorageService.getRedactionLog("dossierId","fileId");
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -1,21 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.realdata;
 
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.*;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import lombok.SneakyThrows;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -34,14 +30,27 @@ import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
+import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.mockito.ArgumentMatchers.*;
-import static org.mockito.Mockito.when;
+import lombok.SneakyThrows;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -134,7 +143,6 @@ public class LiveDataIntegrationTest {
             if (found.isPresent()) {
                 var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
                 type.setEntries(getEntries(typeName, type.getTypeId()));
-                type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId()));
 
                 return type;
             } else {
@@ -188,22 +196,7 @@ public class LiveDataIntegrationTest {
 
         Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
         for (var resource : dictionaryResources) {
-            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) {
-
-                List<String> lines = IOUtils.readLines(resource.getInputStream());
-                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
-
-            }
-        }
-        return new ArrayList<>();
-    }
-
-    @SneakyThrows
-    private List<DictionaryEntry> getFalsePositiveEntries(String typeName, String typeId) {
-
-        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
-        for (var resource : dictionaryResources) {
-            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName)) {
 
                 List<String> lines = IOUtils.readLines(resource.getInputStream());
                 return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/performance/data/test-file-2.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/dictionaries/CBI_author_false_positive.txt

@@ -1 +0,0 @@
-Salt), CGA376944 (S-enantiomer