Do not remove images at reanalysis

Pull request #131 : RED-1276: Fixed annotation position problem for 270° rotated pages
Merge in RED/redaction-service from RED-1276 to master * commit '8375c04829347f43c2151e80dd4f4571b0c2757d': RED-1276: Fixed annotation position problem for 270° rotated pages
2021-04-08 15:22:25 +02:00 · 2021-04-08 10:49:40 +02:00 · 2021-04-08 10:30:51 +02:00 · 2021-03-15 13:37:30 +01:00 · 2021-03-15 13:22:14 +01:00 · 2021-03-15 11:56:38 +01:00
12 changed files with 81 additions and 25 deletions
@@ -5,7 +5,7 @@
    <parent>
        <artifactId>platform-dependency</artifactId>
        <groupId>com.iqser.red</groupId>
-        <version>1.0.3</version>
+        <version>1.0.8</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

@@ -32,7 +32,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.1.7</version>
+                <version>1.2.5</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -73,11 +73,7 @@
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-openfeign</artifactId>
        </dependency>
-        <!-- ribbon must be included because transitive a dependency on it.-->
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-starter-netflix-ribbon</artifactId>
-        </dependency>
+
        <!-- test dependencies -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
@@ -1,10 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;

-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import org.springframework.boot.SpringApplication;
-import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
@@ -12,10 +8,14 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Import;

+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
@Import({DefaultWebMvcConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
-@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
+@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

    public static void main(String[] args) {
@@ -101,10 +101,13 @@ public class TextPositionSequence implements CharSequence {
        }
    }

+
    public float getRotationAdjustedY() {
+
        return textPositions.get(0).getY();
    }

+
    public float getY1() {

        if (textPositions.get(0).getRotation() == 90) {
@@ -202,6 +205,13 @@ public class TextPositionSequence implements CharSequence {
            posXEnd = textPositions.get(0).getYDirAdj() + 2;
            posYInit = getY1();
            posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
+        } else if (textPositions.get(0).getRotation() == 270) {
+            posYInit = textPositions.get(0).getPageHeight() - getX1();
+            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
+                    .getWidth() - textPositions.get(textPositions.size() - 1).getWidth() - 1;
+            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + height;
        } else {
            posXEnd = textPositions.get(textPositions.size() - 1)
                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
@@ -213,5 +223,4 @@ public class TextPositionSequence implements CharSequence {
        return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
    }

-
 }
@@ -9,7 +9,7 @@ import lombok.Data;
@AllArgsConstructor
 public class DictionaryIncrement {

-    private Set<String> values;
+    private Set<DictionaryIncrementValue> values;
    private long dictionaryVersion;

 }
@@ -0,0 +1,13 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class DictionaryIncrementValue {
+
+    private String value;
+    private boolean caseinsensitive;
+
+}
@@ -22,6 +22,7 @@ import com.iqser.red.service.configuration.v1.api.model.TypeResult;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;

@@ -57,12 +58,12 @@ public class DictionaryService {

        long version = updateDictionary(ruleSetId);

-        Set<String> newValues = new HashSet<>();
+        Set<DictionaryIncrementValue> newValues = new HashSet<>();
        List<DictionaryModel> dictionaryModels = dictionariesByRuleSets.get(ruleSetId).getDictionary();
        dictionaryModels.forEach(dictionaryModel -> {
            dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                if (dictionaryEntry.getVersion() > fromVersion) {
-                    newValues.add(dictionaryEntry.getValue());
+                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
                }
            });
        });
@@ -127,7 +128,7 @@ public class DictionaryService {
                .getEntries());

        if (t.isCaseInsensitive()) {
-            entries.forEach(entry -> entry.getValue().toLowerCase(Locale.ROOT));
+            entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
        }
        return entries;
    }
@@ -66,7 +66,7 @@ public class EntityRedactionService {

            Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
            documentEntities.stream().forEach(entity -> {
-                if (dictionary.isHint(entity.getType())) {
+                if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
                    hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>())
                            .add(entity);
                }
@@ -82,9 +82,8 @@ public class ReanalyzeService {
        }

        for (SectionText sectionText : renalyzeRequest.getText().getSectionTexts()) {
-            Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), dictionaryIncrement.getValues(), "find", sectionText
-                    .getHeadline(), sectionText.getSectionNumber(), false);
-            if (!entities.isEmpty()) {
+
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrement.getValues())) {
                sectionsToReanaylse.add(sectionText.getSectionNumber());
            }

@@ -249,7 +248,7 @@ public class ReanalyzeService {
            Iterator<RedactionLogEntry> itty = renalyzeRequest.getRedactionLog().getRedactionLogEntry().iterator();
            while (itty.hasNext()) {
                RedactionLogEntry entry = itty.next();
-                if (sectionsToReanaylse.contains(entry.getSectionNumber()) || entry.getSectionNumber() == 0) {
+                if (sectionsToReanaylse.contains(entry.getSectionNumber()) && !entry.getType().equals("image") || entry.getSectionNumber() == 0) {
                    itty.remove();
                }
            }
@@ -5,12 +5,14 @@ import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;

 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
@@ -20,8 +22,40 @@ import lombok.extern.slf4j.Slf4j;

@Slf4j
@UtilityClass
+@SuppressWarnings("PMD")
 public class EntitySearchUtils {

+
+    public boolean sectionContainsAny(String sectionText, Set<DictionaryIncrementValue> values) {
+
+        String inputString = sectionText.toLowerCase(Locale.ROOT);
+
+        for (DictionaryIncrementValue value : values) {
+
+            String cleanValue = value.getValue().toLowerCase(Locale.ROOT).trim();
+
+            if (cleanValue.length() <= 2) {
+                continue;
+            }
+
+            int startIndex;
+            int stopIndex = 0;
+            do {
+                startIndex = inputString.indexOf(cleanValue, stopIndex);
+                stopIndex = startIndex + cleanValue.length();
+
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+                    if(value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())){
+                        return true;
+                    }
+                }
+            } while (startIndex > -1);
+        }
+        return false;
+    }
+
+
    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
                            boolean local) {

@@ -498,7 +498,7 @@ public class RedactionIntegrationTest {

        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_13_Volume_3CP_A9396G_B-1_2018-09-06.pdf");

        AnalyzeRequest request = AnalyzeRequest.builder()
                .ruleSetId(TEST_RULESET_ID)
@@ -543,8 +543,12 @@ public class RedactionIntegrationTest {
        dictionary.get(AUTHOR).add("physical");
        reanlysisVersions.put("physical", 2L);

-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(2L);
-        when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(AUTHOR));
+        dictionary.get(VERTEBRATE).add("s-metolachlor");
+        reanlysisVersions.put("s-metolachlor", 3L);
+
+        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(3L);
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));

        start = System.currentTimeMillis();
        ReanalyzeResult reanalyzeResult = redactionController.reanalyze(RenalyzeRequest.builder()
Author	SHA1	Message	Date
Dominique Eifländer	e7c24487c7	Do not remove images at reanalysis	2021-04-08 15:22:25 +02:00
Dominique Eiflaender	fc10152145	Pull request #131 : RED-1276: Fixed annotation position problem for 270° rotated pages Merge in RED/redaction-service from RED-1276 to master * commit '8375c04829347f43c2151e80dd4f4571b0c2757d': RED-1276: Fixed annotation position problem for 270° rotated pages	2021-04-08 10:49:40 +02:00
Dominique Eifländer	8375c04829	RED-1276: Fixed annotation position problem for 270° rotated pages	2021-04-08 10:30:51 +02:00
Dominique Eiflaender	57a79719d4	Pull request #130 : RED-1061: Upgraded to newest spring boot/cloud and reenabled actuator metrics Merge in RED/redaction-service from RED-1061 to master * commit '0638877d0afb3b577f7be5467cde1d1b34044772': RED-1061: Upgraded to newest spring boot/cloud and reenabled actuator metrics	2021-03-15 13:37:30 +01:00
Dominique Eifländer	0638877d0a	RED-1061: Upgraded to newest spring boot/cloud and reenabled actuator metrics	2021-03-15 13:22:14 +01:00
Dominique Eiflaender	6c33b7e2ae	Pull request #129 : RED-1170: Do not use non dictionay hints vor local analysis Merge in RED/redaction-service from RED-1170 to master * commit 'aac4d0437e2f9e81f5f4a814dbc86e47b07f57d7': RED-1170: Do not use non dictionay hints vor local analysis	2021-03-15 11:56:38 +01:00
Dominique Eifländer	aac4d0437e	RED-1170: Do not use non dictionay hints vor local analysis	2021-03-15 11:35:40 +01:00
Dominique Eiflaender	93dcd6b9f7	Pull request #128 : Fixed importing caseinsensitive dictionaties Merge in RED/redaction-service from caseInSensFix to master * commit 'bd56364cc3bfa25084069c79bef107869bdc5574': Fixed importing caseinsensitive dictionaties	2021-03-12 15:25:43 +01:00
Dominique Eifländer	bd56364cc3	Fixed importing caseinsensitive dictionaties	2021-03-12 15:08:34 +01:00
Dominique Eiflaender	91942730e8	Pull request #127 : Fixed renalysis for caseinsensitive dictionary entries Merge in RED/redaction-service from caseinsensitiveValues to master * commit 'f4ea236fc587df34eda20ead5762737bca447f3d': Fixed renalysis for caseinsensitive dictionary entries	2021-03-11 14:15:17 +01:00
Dominique Eifländer	f4ea236fc5	Fixed renalysis for caseinsensitive dictionary entries	2021-03-11 13:57:15 +01:00
Dominique Eiflaender	9db74628a4	Pull request #126 : First steps for incremental analysis Merge in RED/redaction-service from incrementAnalysis to master * commit '511092b9e76d58af33dfff90c1133b92e850d47c': First steps for incremental analysis	2021-03-09 09:12:13 +01:00