Do not remove images at reanalysis

Pull request #131 : RED-1276: Fixed annotation position problem for 270° rotated pages
Merge in RED/redaction-service from RED-1276 to master * commit '8375c04829347f43c2151e80dd4f4571b0c2757d': RED-1276: Fixed annotation position problem for 270° rotated pages
2021-04-08 15:22:25 +02:00 · 2021-04-08 10:49:40 +02:00 · 2021-04-08 10:30:51 +02:00 · 2021-03-15 13:37:30 +01:00 · 2021-03-15 13:22:14 +01:00 · 2021-03-15 11:56:38 +01:00
14 changed files with 182 additions and 74 deletions
@@ -5,7 +5,7 @@
    <parent>
        <artifactId>platform-dependency</artifactId>
        <groupId>com.iqser.red</groupId>
-        <version>1.0.3</version>
+        <version>1.0.8</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

@@ -32,7 +32,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.1.7</version>
+                <version>1.2.5</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -20,7 +20,7 @@
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>configuration-service-api-v1</artifactId>
-            <version>2.0.0</version>
+            <version>2.2.9</version>
        </dependency>
        <dependency>
            <groupId>org.drools</groupId>
@@ -73,11 +73,7 @@
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-openfeign</artifactId>
        </dependency>
-        <!-- ribbon must be included because transitive a dependency on it.-->
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-starter-netflix-ribbon</artifactId>
-        </dependency>
+
        <!-- test dependencies -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
@@ -1,10 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;

-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import org.springframework.boot.SpringApplication;
-import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
@@ -12,10 +8,14 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Import;

+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
@Import({DefaultWebMvcConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
-@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
+@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

    public static void main(String[] args) {
@@ -101,10 +101,13 @@ public class TextPositionSequence implements CharSequence {
        }
    }

+
    public float getRotationAdjustedY() {
+
        return textPositions.get(0).getY();
    }

+
    public float getY1() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -202,6 +205,13 @@ public class TextPositionSequence implements CharSequence {
            posXEnd = textPositions.get(0).getYDirAdj() + 2;
            posYInit = getY1();
            posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
+        } else if (textPositions.get(0).getRotation() == 270) {
+            posYInit = textPositions.get(0).getPageHeight() - getX1();
+            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
+                    .getWidth() - textPositions.get(textPositions.size() - 1).getWidth() - 1;
+            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + height;
        } else {
            posXEnd = textPositions.get(textPositions.size() - 1)
                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
@@ -213,5 +223,4 @@ public class TextPositionSequence implements CharSequence {
        return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
    }

-
 }
@@ -9,7 +9,7 @@ import lombok.Data;
@AllArgsConstructor
 public class DictionaryIncrement {

-    private Set<String> values;
+    private Set<DictionaryIncrementValue> values;
    private long dictionaryVersion;

 }
@@ -0,0 +1,13 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class DictionaryIncrementValue {
+
+    private String value;
+    private boolean caseinsensitive;
+
+}
@@ -3,6 +3,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;

 import java.io.Serializable;
 import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;

 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -17,11 +20,12 @@ public class DictionaryModel implements Serializable {
    private boolean caseInsensitive;
    private boolean hint;
    private boolean recommendation;
-    private Set<String> entries;
+    private Set<DictionaryEntry> entries;
    private Set<String> localEntries;

    public Set<String> getValues(boolean local){
-        return local ? localEntries : entries;
+        return local ? localEntries : entries.stream().filter(e -> !e.isDeleted()).map(e-> e.getValue()).collect(Collectors
+                .toSet());
    }

 }
@@ -1,43 +1,46 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import com.iqser.red.service.configuration.v1.api.model.Colors;
-import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
-import com.iqser.red.service.configuration.v1.api.model.TypeResult;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
-import feign.FeignException;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.SerializationUtils;
-import org.springframework.stereotype.Service;
-
 import java.awt.Color;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;

+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.SerializationUtils;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.configuration.v1.api.model.Colors;
+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
+
+import feign.FeignException;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
@Service
@RequiredArgsConstructor
 public class DictionaryService {

-
    private final DictionaryClient dictionaryClient;

-
    private Map<String, DictionaryRepresentation> dictionariesByRuleSets = new HashMap<>();


-    public void updateDictionary(String ruleSetId) {
+    public long updateDictionary(String ruleSetId) {

        long version = dictionaryClient.getVersion(ruleSetId);

@@ -46,14 +49,26 @@ public class DictionaryService {
        if (foundDictionary == null || version > foundDictionary.getDictionaryVersion()) {
            updateDictionaryEntry(ruleSetId, version);
        }
+
+        return version;
    }


-    public DictionaryIncrement getDictionaryIncrements(String ruleSetId, long fromVersion){
+    public DictionaryIncrement getDictionaryIncrements(String ruleSetId, long fromVersion) {

-        updateDictionary(ruleSetId);
+        long version = updateDictionary(ruleSetId);

-        return new DictionaryIncrement(Set.of("Properties"), 1L);
+        Set<DictionaryIncrementValue> newValues = new HashSet<>();
+        List<DictionaryModel> dictionaryModels = dictionariesByRuleSets.get(ruleSetId).getDictionary();
+        dictionaryModels.forEach(dictionaryModel -> {
+            dictionaryModel.getEntries().forEach(dictionaryEntry -> {
+                if (dictionaryEntry.getVersion() > fromVersion) {
+                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                }
+            });
+        });
+
+        return new DictionaryIncrement(newValues, version);
    }


@@ -72,7 +87,6 @@ public class DictionaryService {
                        .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                        .collect(Collectors.toList());

-
                dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));

                Colors colors = dictionaryClient.getColors(ruleSetId);
@@ -95,6 +109,7 @@ public class DictionaryService {


    public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) {
+
        dictionary.getDictionaryModels().forEach(dm -> {
            if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
                dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false);
@@ -107,17 +122,15 @@ public class DictionaryService {
    }


-    private Set<String> convertEntries(TypeResult t) {
+    private Set<DictionaryEntry> convertEntries(TypeResult t) {
+
+        Set<DictionaryEntry> entries = new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
+                .getEntries());

        if (t.isCaseInsensitive()) {
-            return dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
-                    .getEntries()
-                    .stream()
-                    .map(String::toLowerCase)
-                    .collect(Collectors.toSet());
-        } else {
-            return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId()).getEntries());
+            entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
        }
+        return entries;
    }


@@ -157,6 +170,7 @@ public class DictionaryService {
        return false;
    }

+
    public boolean isRecommendation(String type, String ruleSetId) {

        DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
@@ -168,6 +182,7 @@ public class DictionaryService {


    public Dictionary getDeepCopyDictionary(String ruleSetId) {
+
        List<DictionaryModel> copy = new ArrayList<>();

        var representation = dictionariesByRuleSets.get(ruleSetId);
@@ -179,15 +194,22 @@ public class DictionaryService {
        return new Dictionary(copy, representation.getDictionaryVersion());
    }

+
    public float[] getRequestRemoveColor(String ruleSetId) {
+
        return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
    }

+
    public float[] getNotRedactedColor(String ruleSetId) {
+
        return dictionariesByRuleSets.get(ruleSetId).getNotRedactedColor();
    }

+
    public float[] getRequestAddColor(String ruleSetId) {
+
        return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
    }
+
 }
@@ -66,7 +66,7 @@ public class EntityRedactionService {

            Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
            documentEntities.stream().forEach(entity -> {
-                if (dictionary.isHint(entity.getType())) {
+                if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
                    hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>())
                            .add(entity);
                }
@@ -82,9 +82,8 @@ public class ReanalyzeService {
        }

        for (SectionText sectionText : renalyzeRequest.getText().getSectionTexts()) {
-            Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), dictionaryIncrement.getValues(), "find", sectionText
-                    .getHeadline(), sectionText.getSectionNumber(), false);
-            if (!entities.isEmpty()) {
+
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrement.getValues())) {
                sectionsToReanaylse.add(sectionText.getSectionNumber());
            }

@@ -249,7 +248,7 @@ public class ReanalyzeService {
            Iterator<RedactionLogEntry> itty = renalyzeRequest.getRedactionLog().getRedactionLogEntry().iterator();
            while (itty.hasNext()) {
                RedactionLogEntry entry = itty.next();
-                if (sectionsToReanaylse.contains(entry.getSectionNumber()) || entry.getSectionNumber() == 0) {
+                if (sectionsToReanaylse.contains(entry.getSectionNumber()) && !entry.getType().equals("image") || entry.getSectionNumber() == 0) {
                    itty.remove();
                }
            }
@@ -5,12 +5,14 @@ import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;

 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
@@ -20,8 +22,40 @@ import lombok.extern.slf4j.Slf4j;

@Slf4j
@UtilityClass
+@SuppressWarnings("PMD")
 public class EntitySearchUtils {

+
+    public boolean sectionContainsAny(String sectionText, Set<DictionaryIncrementValue> values) {
+
+        String inputString = sectionText.toLowerCase(Locale.ROOT);
+
+        for (DictionaryIncrementValue value : values) {
+
+            String cleanValue = value.getValue().toLowerCase(Locale.ROOT).trim();
+
+            if (cleanValue.length() <= 2) {
+                continue;
+            }
+
+            int startIndex;
+            int stopIndex = 0;
+            do {
+                startIndex = inputString.indexOf(cleanValue, stopIndex);
+                stopIndex = startIndex + cleanValue.length();
+
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+                    if(value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())){
+                        return true;
+                    }
+                }
+            } while (startIndex > -1);
+        }
+        return false;
+    }
+
+
    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
                            boolean local) {

@@ -45,6 +45,7 @@ import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
+import org.mockito.MockitoAnnotations;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.context.TestConfiguration;
@@ -55,6 +56,7 @@ import org.springframework.test.context.junit4.SpringRunner;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.configuration.v1.api.model.Colors;
+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
@@ -136,6 +138,7 @@ public class RedactionIntegrationTest {
    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
    private final Map<String, Integer> rankTypeMap = new HashMap<>();
    private final Colors colors = new Colors();
+    private final Map<String, Long> reanlysisVersions = new HashMap<>();

    private final static String TEST_RULESET_ID = "123";

@@ -400,7 +403,7 @@ public class RedactionIntegrationTest {

        return DictionaryResponse.builder()
                .hexColor(typeColorMap.get(type))
-                .entries(dictionary.get(type))
+                .entries(toDictionaryEntry(dictionary.get(type)))
                .isHint(hintTypeMap.get(type))
                .isCaseInsensitive(caseInSensitiveMap.get(type))
                .isRecommendation(recommendationTypeMap.get(type))
@@ -409,6 +412,15 @@ public class RedactionIntegrationTest {
    }


+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries){
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, false));
+        });
+        return dictionaryEntries;
+    }
+
+
    @Test
    public void noExceptionShouldBeThrownForAnyFiles() throws IOException {

@@ -486,7 +498,7 @@ public class RedactionIntegrationTest {

        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_13_Volume_3CP_A9396G_B-1_2018-09-06.pdf");

        AnalyzeRequest request = AnalyzeRequest.builder()
                .ruleSetId(TEST_RULESET_ID)
@@ -525,8 +537,18 @@ public class RedactionIntegrationTest {
        assertThat(correctFound).isEqualTo(result.getRedactionLog().getRedactionLogEntry().size());


-        dictionary.get(AUTHOR).add("Properties");
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(1L);
+        dictionary.get(AUTHOR).add("properties");
+        reanlysisVersions.put("properties", 1L);
+
+        dictionary.get(AUTHOR).add("physical");
+        reanlysisVersions.put("physical", 2L);
+
+        dictionary.get(VERTEBRATE).add("s-metolachlor");
+        reanlysisVersions.put("s-metolachlor", 3L);
+
+        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(3L);
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));

        start = System.currentTimeMillis();
        ReanalyzeResult reanalyzeResult = redactionController.reanalyze(RenalyzeRequest.builder()
@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

 import com.iqser.red.service.configuration.v1.api.model.Colors;
+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
@@ -129,12 +130,12 @@ public class EntityRedactionServiceTest {
                .build();

        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
+                .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H.")))
                .build();
        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
+                .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")))
                .build();
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);

@@ -162,12 +163,12 @@ public class EntityRedactionServiceTest {
                .build();

        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
+                .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H.")))
                .build();
        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
+                .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")))
                .build();
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
@@ -191,11 +192,11 @@ public class EntityRedactionServiceTest {
                " Supplement - Identity of the active substance - Reference list.pdf");
        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                .build();
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                .build();
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
@@ -228,15 +229,15 @@ public class EntityRedactionServiceTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                .build();
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                .build();
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))))
                .build();
        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
@@ -297,11 +298,11 @@ public class EntityRedactionServiceTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                .build();
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                .build();
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
@@ -346,7 +347,7 @@ public class EntityRedactionServiceTest {
                .build();
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(authorResponse);
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))))
                .build();
        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
@@ -367,13 +368,13 @@ public class EntityRedactionServiceTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Header Propagation.pdf");

        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Bissig R.", "Thanei P."))
+                .entries(toDictionaryEntry(Arrays.asList("Bissig R.", "Thanei P.")))
                .build();

        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
+                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                .build();
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);

@@ -392,13 +393,13 @@ public class EntityRedactionServiceTest {
        pdfFileResource = new ClassPathResource("files/Minimal Examples/Header Propagation2.pdf");

        dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C."))
+                .entries(toDictionaryEntry(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C.")))
                .build();

        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
+                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                .build();
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);

@@ -419,13 +420,13 @@ public class EntityRedactionServiceTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Empty Tabular Data.pdf");

        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Aldershof S."))
+                .entries(toDictionaryEntry(Collections.singletonList("Aldershof S.")))
                .build();

        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
+                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                .build();
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);

@@ -517,4 +518,12 @@ public class EntityRedactionServiceTest {
        }
    }

+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries){
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(new DictionaryEntry(entry, 1L, false));
+        });
+        return dictionaryEntries;
+    }
+
 }
Author	SHA1	Message	Date
Dominique Eifländer	e7c24487c7	Do not remove images at reanalysis	2021-04-08 15:22:25 +02:00
Dominique Eiflaender	fc10152145	Pull request #131 : RED-1276: Fixed annotation position problem for 270° rotated pages Merge in RED/redaction-service from RED-1276 to master * commit '8375c04829347f43c2151e80dd4f4571b0c2757d': RED-1276: Fixed annotation position problem for 270° rotated pages	2021-04-08 10:49:40 +02:00
Dominique Eifländer	8375c04829	RED-1276: Fixed annotation position problem for 270° rotated pages	2021-04-08 10:30:51 +02:00
Dominique Eiflaender	57a79719d4	Pull request #130 : RED-1061: Upgraded to newest spring boot/cloud and reenabled actuator metrics Merge in RED/redaction-service from RED-1061 to master * commit '0638877d0afb3b577f7be5467cde1d1b34044772': RED-1061: Upgraded to newest spring boot/cloud and reenabled actuator metrics	2021-03-15 13:37:30 +01:00
Dominique Eifländer	0638877d0a	RED-1061: Upgraded to newest spring boot/cloud and reenabled actuator metrics	2021-03-15 13:22:14 +01:00
Dominique Eiflaender	6c33b7e2ae	Pull request #129 : RED-1170: Do not use non dictionay hints vor local analysis Merge in RED/redaction-service from RED-1170 to master * commit 'aac4d0437e2f9e81f5f4a814dbc86e47b07f57d7': RED-1170: Do not use non dictionay hints vor local analysis	2021-03-15 11:56:38 +01:00
Dominique Eifländer	aac4d0437e	RED-1170: Do not use non dictionay hints vor local analysis	2021-03-15 11:35:40 +01:00
Dominique Eiflaender	93dcd6b9f7	Pull request #128 : Fixed importing caseinsensitive dictionaties Merge in RED/redaction-service from caseInSensFix to master * commit 'bd56364cc3bfa25084069c79bef107869bdc5574': Fixed importing caseinsensitive dictionaties	2021-03-12 15:25:43 +01:00
Dominique Eifländer	bd56364cc3	Fixed importing caseinsensitive dictionaties	2021-03-12 15:08:34 +01:00
Dominique Eiflaender	91942730e8	Pull request #127 : Fixed renalysis for caseinsensitive dictionary entries Merge in RED/redaction-service from caseinsensitiveValues to master * commit 'f4ea236fc587df34eda20ead5762737bca447f3d': Fixed renalysis for caseinsensitive dictionary entries	2021-03-11 14:15:17 +01:00
Dominique Eifländer	f4ea236fc5	Fixed renalysis for caseinsensitive dictionary entries	2021-03-11 13:57:15 +01:00
Dominique Eiflaender	9db74628a4	Pull request #126 : First steps for incremental analysis Merge in RED/redaction-service from incrementAnalysis to master * commit '511092b9e76d58af33dfff90c1133b92e850d47c': First steps for incremental analysis	2021-03-09 09:12:13 +01:00
Dominique Eifländer	511092b9e7	First steps for incremental analysis	2021-03-08 16:31:37 +01:00