Fixed table offset bug

Merge in RED/redaction-service from RED-864 to master

* commit 'e43bd1b71134ce9d34cc521459240c10606de7fc':
  RED-864, Added isDictionaryEntry to redactionLog. Fixed order of dictionary types

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -32,5 +32,6 @@ public class RedactionLogEntry {
     private boolean manual;
     private Status status;
     private ManualRedactionType manualRedactionType;
+    private boolean isDictionaryEntry;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/CellValue.java

@@ -1,5 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import java.util.Iterator;
+import java.util.List;
+
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
@@ -9,9 +12,9 @@ import lombok.Value;
 @Value
 public class CellValue {
 
-    TextBlock textBlock;
+    private List<TextBlock> textBlocks;
 
-    int rowSpanStart;
+    private int rowSpanStart;
 
 
     @Override
@@ -19,17 +22,24 @@ public class CellValue {
 
         StringBuilder sb = new StringBuilder();
 
-        TextPositionSequence previous = null;
-        for (TextPositionSequence word : textBlock.getSequences()) {
-            if (previous != null) {
-                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
-                    sb.append('\n');
-                } else {
-                    sb.append(' ');
+        Iterator<TextBlock> itty = textBlocks.iterator();
+        while (itty.hasNext()) {
+            TextBlock textBlock = itty.next();
+            TextPositionSequence previous = null;
+            for (TextPositionSequence word : textBlock.getSequences()) {
+                if (previous != null) {
+                    if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                        sb.append('\n');
+                    } else {
+                        sb.append(' ');
+                    }
                 }
+                sb.append(word.toString());
+                previous = word;
+            }
+            if (itty.hasNext()) {
+                sb.append(' ');
             }
-            sb.append(word.toString());
-            previous = word;
         }
 
         return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java

@@ -1,27 +1,33 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.Getter;
 
 @Data
-@AllArgsConstructor
 public class Dictionary {
 
     public static final String RECOMMENDATION_PREFIX = "recommendation_";
 
     @Getter
     private List<DictionaryModel> dictionaryModels;
-    private Map<String, DictionaryModel> localAccessMap;
+    private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
 
     @Getter
     private long version;
 
 
+    public Dictionary(List<DictionaryModel> dictionaryModels, long dictionaryVersion){
+        this.dictionaryModels = dictionaryModels;
+        this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
+        this.version = dictionaryVersion;
+    }
+
+
     public boolean isRecommendation(String type) {
 
         DictionaryModel model = localAccessMap.get(type);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
 
+import java.io.Serializable;
 import java.util.Set;
 
 import lombok.AllArgsConstructor;
@@ -8,7 +9,7 @@ import lombok.Data;
 
 @Data
 @AllArgsConstructor
-public class DictionaryModel {
+public class DictionaryModel implements Serializable {
 
     private String type;
     private int rank;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -12,7 +12,7 @@ import lombok.EqualsAndHashCode;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity {
 
-    @EqualsAndHashCode.Include
+
     private final String word;
     private final String type;
     private boolean redaction;
@@ -20,7 +20,10 @@ public class Entity {
     private String legalBasis;
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();
     private List<TextPositionSequence> targetSequences;
+
+    @EqualsAndHashCode.Include
     private Integer start;
+    @EqualsAndHashCode.Include
     private Integer end;
 
     @EqualsAndHashCode.Include
@@ -30,8 +33,10 @@ public class Entity {
     @EqualsAndHashCode.Include
     private int sectionNumber;
 
+    private boolean isDictionaryEntry;
 
-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis) {
+
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry) {
 
         this.word = word;
         this.type = type;
@@ -42,10 +47,11 @@ public class Entity {
         this.matchedRule = matchedRule;
         this.sectionNumber = sectionNumber;
         this.legalBasis = legalBasis;
+        this.isDictionaryEntry = isDictionaryEntry;
     }
 
 
-    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {
+    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry) {
 
         this.word = word;
         this.type = type;
@@ -53,6 +59,7 @@ public class Entity {
         this.end = end;
         this.headline = headline;
         this.sectionNumber = sectionNumber;
+        this.isDictionaryEntry = isDictionaryEntry;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -51,8 +51,7 @@ public class Section {
         String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
 
         return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
-                .getTextBlock()
-                .getText()
+                .toString()
                 .equals(value);
     }
 
@@ -230,7 +229,7 @@ public class Section {
 
             if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(text.charAt(startIndex - 1)) || isSeparator(text
                     .charAt(startIndex - 1))) && (stopIndex == text.length() || isSeparator(text.charAt(stopIndex)))) {
-                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber));
+                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber, false));
             }
         } while (startIndex > -1);
 
@@ -291,11 +290,11 @@ public class Section {
         } else {
             String word = value.toString();
 
-            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber);
+            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false);
             entity.setRedaction(redact);
             entity.setMatchedRule(ruleNumber);
             entity.setRedactionReason(reason);
-            entity.setTargetSequences(value.getTextBlock()
+            entity.setTargetSequences(value.getTextBlocks().get(0)
                     .getSequences()); // Make sure no other cells with same content are highlighted
             entity.setLegalBasis(legalBasis);
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -11,6 +11,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.configuration.v1.api.model.Colors;
@@ -168,8 +169,14 @@ public class DictionaryService {
     }
 
 
-    public Dictionary getThreadSafeDictionary(){
-        return new Dictionary(new ArrayList<>(dictionary), new HashMap<>(localAccessMap), dictionaryVersion);
+    public Dictionary getDeepCopyDictionary(){
+        List<DictionaryModel> copy = new ArrayList<>();
+
+        dictionary.forEach(dm -> {
+            copy.add(SerializationUtils.clone(dm));
+        });
+
+        return new Dictionary(copy, dictionaryVersion);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -32,7 +32,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @RequiredArgsConstructor
 public class EntityRedactionService {
@@ -47,7 +49,7 @@ public class EntityRedactionService {
         droolsExecutionService.updateRules();
         long rulesVersion = droolsExecutionService.getRulesVersion();
 
-        Dictionary dictionary = dictionaryService.getThreadSafeDictionary();
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary();
 
         Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, manualRedactions, dictionary, false, null));
 
@@ -81,7 +83,7 @@ public class EntityRedactionService {
                         .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
                         .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                 .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
-                                .getLegalBasis()));
+                                .getLegalBasis(), entity.isDictionaryEntry()));
             }
         }
 
@@ -124,9 +126,9 @@ public class EntityRedactionService {
                                     .replaceAll("\n", "")
                                     .replaceAll(" ", "")
                                     .replaceAll("-", "");
-                            tabularData.put(headerName, new CellValue(cell.getTextBlocks().get(0), cellStart));
+                            tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
                         });
-                        start = start + cell.toString().length() + 1; // include automatically appended white space
+                        start = start + cell.toString().length() + 1;
                         for (TextBlock textBlock : cell.getTextBlocks()) {
                             searchableRow.addAll(textBlock.getSequences());
                         }
@@ -182,7 +184,16 @@ public class EntityRedactionService {
                     });
                 } else {
                     analysedRowSection.getLocalDictionaryAdds().get(key).forEach( value -> {
-                            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
+
+                        if(dictionary.getLocalAccessMap().get(key) == null){
+                            log.warn("Dictionary {} is null", key);
+                        }
+
+                        if(dictionary.getLocalAccessMap().get(key).getLocalEntries() == null){
+                            log.warn("Dictionary {} localEntries is null", key);
+                        }
+
+                        dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
                     });
                 }
             });
@@ -221,9 +232,9 @@ public class EntityRedactionService {
         String lowercaseInputString = searchableString.toLowerCase();
         for (DictionaryModel model : dictionary) {
             if (model.isCaseInsensitive()) {
-                found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber));
+                found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
             } else {
-                found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber));
+                found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
             }
         }
         removeEntitiesContainedInLarger(found);
@@ -232,7 +243,7 @@ public class EntityRedactionService {
     }
 
 
-    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber) {
+    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber, boolean local) {
 
         Set<Entity> found = new HashSet<>();
         for (String value : values) {
@@ -244,7 +255,7 @@ public class EntityRedactionService {
 
                 if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
                         .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber));
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local));
                 }
             } while (startIndex > -1);
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;
 
 import java.awt.geom.Point2D;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@@ -41,7 +42,9 @@ public class Cell extends Rectangle {
 
         StringBuilder sb = new StringBuilder();
 
-        for (TextBlock textBlock : textBlocks) {
+        Iterator<TextBlock> itty = textBlocks.iterator();
+        while (itty.hasNext()) {
+            TextBlock textBlock = itty.next();
             TextPositionSequence previous = null;
             for (TextPositionSequence word : textBlock.getSequences()) {
                 if (previous != null) {
@@ -54,6 +57,9 @@ public class Cell extends Rectangle {
                 sb.append(word.toString());
                 previous = word;
             }
+            if (itty.hasNext()) {
+                sb.append(' ');
+            }
         }
 
         return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java

@@ -256,6 +256,7 @@ public class AnnotationHighlightService {
                 .manual(true)
                 .status(manualRedactionEntry.getStatus())
                 .manualRedactionType(ManualRedactionType.ADD)
+                .isDictionaryEntry(false)
                 .build();
     }
 
@@ -274,6 +275,7 @@ public class AnnotationHighlightService {
                 .section(entity.getHeadline())
                 .sectionNumber(entity.getSectionNumber())
                 .matchedRule(entity.getMatchedRule())
+                .isDictionaryEntry(entity.isDictionaryEntry())
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -382,7 +382,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("redactionTest");
         long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
 
         RedactionRequest request = RedactionRequest.builder()
                 .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -392,7 +392,7 @@ public class RedactionIntegrationTest {
         RedactionResult result = redactionController.redact(request);
 
         result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
-            if(entry.isRecommendation()){
+            if(entry.isDictionaryEntry()){
                 System.out.println(entry.getValue());
             }
         });

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java

@@ -104,8 +104,8 @@ public class EntityRedactionServiceTest {
     public void testNestedEntitiesRemoval() {
 
         Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false);
         entities.add(nested);
         entities.add(nesting);
         entityRedactionService.removeEntitiesContainedInLarger(entities);
@@ -404,7 +404,7 @@ public class EntityRedactionServiceTest {
             entityRedactionService.processDocument(classifiedDoc, null);
             assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
             assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3);
-            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(8);
+            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9);
         }
     }