Fixed duplicated redaction/RedactionLog entries

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,16 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 import lombok.Data;
-import lombok.EqualsAndHashCode;
 
 @Data
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity {
 
-    @EqualsAndHashCode.Include
     private final String word;
     private final String type;
     private boolean redaction;
@@ -18,17 +17,11 @@ public class Entity {
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();
     private Integer start;
     private Integer end;
-
-    @EqualsAndHashCode.Include
     private String headline;
     private int matchedRule;
-
-    @EqualsAndHashCode.Include
     private int sectionNumber;
 
-
     public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber) {
-
         this.word = word;
         this.type = type;
         this.redaction = redaction;
@@ -39,9 +32,7 @@ public class Entity {
         this.sectionNumber = sectionNumber;
     }
 
-
     public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {
-
         this.word = word;
         this.type = type;
         this.start = start;
@@ -50,4 +41,26 @@ public class Entity {
         this.sectionNumber = sectionNumber;
     }
 
+
+    @Override
+    public boolean equals(Object o) {
+
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        Entity entity = (Entity) o;
+        return sectionNumber == entity.sectionNumber && Objects.equals(word, entity.word) && Objects.equals(type, entity.type) && Objects
+                .equals(headline, entity.headline);
+    }
+
+
+    @Override
+    public int hashCode() {
+
+        return Objects.hash(word, type, headline, sectionNumber);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityPositionSequence.java

@@ -2,24 +2,44 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.UUID;
 
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
-import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;
 
+
 @Data
 @RequiredArgsConstructor
 @AllArgsConstructor
-@EqualsAndHashCode
 public class EntityPositionSequence {
 
-    @EqualsAndHashCode.Exclude
     private List<TextPositionSequence> sequences = new ArrayList<>();
     private int pageNumber;
     private final UUID id;
 
+
+    @Override
+    public boolean equals(Object o) {
+
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        EntityPositionSequence that = (EntityPositionSequence) o;
+        return pageNumber == that.pageNumber && Objects.equals(id, that.id);
+    }
+
+
+    @Override
+    public int hashCode() {
+
+        return Objects.hash(pageNumber, id);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -34,8 +34,7 @@ public class Section {
     }
 
 
-    public boolean headlineContainsWord(String word) {
-
+    public boolean headlineContainsWord(String word){
         return StringUtils.containsIgnoreCase(headline, word);
     }
 
@@ -66,15 +65,11 @@ public class Section {
 
     public void redactLineAfter(String start, String asType, int ruleNumber, String reason) {
 
-        String[] values = StringUtils.substringsBetween(text, start, "\n");
+        String value = StringUtils.substringBetween(text, start, "\n");
 
-        if (values != null) {
-            for (String value : values) {
-                if (StringUtils.isNotBlank(value)) {
-                    Set<Entity> found = findEntity(value.trim(), asType);
-                    entities.addAll(found);
-                }
-            }
+        if (value != null) {
+            Set<Entity> found = findEntity(value.trim(), asType);
+            entities.addAll(found);
         }
 
         // TODO No need to iterate
@@ -91,15 +86,11 @@ public class Section {
 
     public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason) {
 
-        String[] values = StringUtils.substringsBetween(searchText, start, stop);
+        String value = StringUtils.substringBetween(searchText, start, stop);
 
-        if (values != null) {
-            for (String value : values) {
-                if (value != null && StringUtils.isNotBlank(value)) {
-                    Set<Entity> found = findEntity(value.trim(), asType);
-                    entities.addAll(found);
-                }
-            }
+        if (value != null) {
+            Set<Entity> found = findEntity(value.trim(), asType);
+            entities.addAll(found);
         }
 
         // TODO No need to iterate

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -73,7 +73,7 @@ public class DictionaryService {
                         .filter(TypeResult::isCaseInsensitive)
                         .map(TypeResult::getType)
                         .collect(Collectors.toList());
-                dictionary = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, this::convertEntries));
+                dictionary = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, s -> convertEntries(s)));
                 defaultColor = dictionaryClient.getDefaultColor().getColor();
             }
         } catch (FeignException e) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -103,9 +103,9 @@ public class EntityRedactionService {
 
     private Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text) {
 
-        removeEntitiesContainedInLarger(entities);
+        Set<Entity> cleanEntities = removeEntitiesContainedInLarger(entities);
 
-        for (Entity entity : entities) {
+        for (Entity entity : cleanEntities) {
             if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
                 entity.setPositionSequences(text.getSequences(entity.getWord(), true));
             } else {
@@ -113,7 +113,7 @@ public class EntityRedactionService {
             }
         }
 
-        return entities;
+        return cleanEntities;
     }
 
 
@@ -132,10 +132,7 @@ public class EntityRedactionService {
             }
         }
 
-        removeEntitiesContainedInLarger(found);
-
-        return found;
-
+        return removeEntitiesContainedInLarger(found);
     }
 
 
@@ -165,7 +162,7 @@ public class EntityRedactionService {
     }
 
 
-    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+    public Set<Entity> removeEntitiesContainedInLarger(Set<Entity> entities) {
 
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
@@ -177,6 +174,7 @@ public class EntityRedactionService {
             }
         }
         entities.removeAll(wordsToRemove);
+        return entities;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -19,7 +19,6 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
@@ -48,7 +47,6 @@ import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
-@Ignore
 @RunWith(SpringRunner.class)
 @SpringBootTest(webEnvironment = DEFINED_PORT)
 public class RedactionIntegrationTest {
@@ -60,7 +58,6 @@ public class RedactionIntegrationTest {
     private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
     private static final String REDACTION_INDICATOR = "redaction_indicator";
     private static final String HINT_ONLY = "hint_only";
-    private static final String MUST_REDACT = "must_redact";
 
     @Autowired
     private RedactionController redactionController;
@@ -114,7 +111,6 @@ public class RedactionIntegrationTest {
         when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
         when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
         when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT)).thenReturn(getDictionaryResponse(MUST_REDACT));
         when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor(new float[]{1f, 0.502f, 0f}));
     }
 
@@ -151,11 +147,6 @@ public class RedactionIntegrationTest {
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/must_redact.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
     }
 
 
@@ -173,7 +164,6 @@ public class RedactionIntegrationTest {
         typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{0.8f, 0, 0.8f});
         typeColorMap.put(REDACTION_INDICATOR, new float[]{1, 0.502f, 0.1f});
         typeColorMap.put(HINT_ONLY, new float[]{0.8f, 1, 0.8f});
-        typeColorMap.put(MUST_REDACT, new float[]{1, 0, 0});
 
         hintTypeMap.put(VERTEBRATES_CODE, true);
         hintTypeMap.put(ADDRESS_CODE, false);
@@ -181,7 +171,6 @@ public class RedactionIntegrationTest {
         hintTypeMap.put(NO_REDACTION_INDICATOR, true);
         hintTypeMap.put(REDACTION_INDICATOR, true);
         hintTypeMap.put(HINT_ONLY, true);
-        hintTypeMap.put(MUST_REDACT, true);
 
         caseInSensitiveMap.put(VERTEBRATES_CODE, true);
         caseInSensitiveMap.put(ADDRESS_CODE, false);
@@ -189,7 +178,6 @@ public class RedactionIntegrationTest {
         caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
         caseInSensitiveMap.put(REDACTION_INDICATOR, true);
         caseInSensitiveMap.put(HINT_ONLY, true);
-        caseInSensitiveMap.put(MUST_REDACT, true);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java

@@ -1,50 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-
-@RunWith(SpringRunner.class)
-@SpringBootTest
-public class EntityRedactionServiceTest {
-
-    @MockBean
-    private KieContainer kieContainer;
-
-    @MockBean
-    private DroolsExecutionService droolsExecutionService;
-
-    @MockBean
-    private DictionaryService dictionaryService;
-
-    @Autowired
-    private EntityRedactionService entityRedactionService;
-
-
-    @Test
-    public void testNestedEntitiesRemoval() {
-
-        Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0);
-        entities.add(nested);
-        entities.add(nesting);
-        entityRedactionService.removeEntitiesContainedInLarger(entities);
-
-        assertThat(entities.size()).isEqualTo(1);
-        assertThat(entities).contains(nesting);
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt

@@ -1,3 +0,0 @@
-Batches Produced at
-CTL
-for determination of residues

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt

@@ -1,63 +1,48 @@
-a. sylvaticus
+Vulpes vulpes
 african clawed frog
 agalychnis callidryas
-albino rat
-american bullfrog tadpole
-american toad
 amphibian
 amphibians
+American bullfrog tadpole
+american toad
 anad platyrhynchos
-anas platyrhynchos
+Anas platyrhynchos
 anuran
 anurans
-apodemus
-apodemus flavicollis
 apodemus syl vaticus
-apodemus sylvaticus
-arvicola terrestris
 avian
-bank vole
 bird
 birds
 bluegill
 bluegill sunfish
 bobwhite
 bobwhite quail
-brachydanio rerio
-brown hare
-bufo americanus
 bullfrog
+Bufo americanus
+brachydanio rerio
 canary
 carassius carassius
 carp
-catesbeiana
 catfish
 cattle
 cattles
 channel catfish
+Chinook
 chicken
-chinese hamster
-chinese hamsters
-chinook
-coho salmon
+Colinus virginianus
 colinus virginianus
-columba palumbus
-columbidae
-common carp
-common vole
+Common carp
 coturnix japonica
+Coturnix japonica
 cow
 cows
-crocidura russula
-crucian carp
+Crucian carp
 cyprinodon variegatus
 cyprinus carpio
 dog
 dogs
 duck
 ducks
-european brown hare
-european rabbit
 fathead minnow
 fish
 fishes
@@ -71,121 +56,175 @@ galaxias truttaceus
 gasterosteus aculeatus
 goat
 goats
-greater white-toothed shrew
 guinea
 guinea pig
 guinea pigs
-guinea-pigs
-guppy
+Guppy
 hamster
 hamsters
 hen
 hens
-house mouse
-hyla versicolor
+Hyla versicolor
 ictalurus melas
 ictalurus punctatus
 japanese quail
 japonica
-kisutch
-lagomorph
 lebistes reticulatus
 leiostomus xanthurus
 leisostomus xanthurus
 lepomis macrochirus
-lepus europaeus
-limnocharis
-limnodynastes
-limnodynastes tasmaniensis
 livestock
 livestocks
-mallard
 mallard duck
 mammal
-mammalian
 mammals
-marten
-martes
+Mammalian
 mice
-microtus
-microtus agrestis
-microtus arvalis
-microtus subterraneus
 midwestern anurans
-minnow
-minnows
 monkey
 mouse
-mus musculus
-myodes glareolus
 northern bobwhite
-o. cuniculus
 o. mykiss
-o. tshawytscha
-oncorhynchus
-oncorhynchus mykiss
-oncorhynchus tshawytscha
-oryctolagus cuniculus
+Oncorhynchus mykiss
+Oncorhynchus
+O. mykiss
 oryzias melastigma
 oryzias melastigma larvae
 p. promelas
 pagrus major
-palumbus
 pig
 pigeon
 pigeons
-pigs
-pimephales promela
 pimephales promelas
+Pseudacris triseriata
 poecilia reticulata
 poultry
-pseudacris
-pseudacris triseriata
 quail
-r. catesbeiana
-rabbit
-rabbits
 rainbow trout
+Rana limnocharis
 rana
-rana catesbeiana
-rana limnocharis
+limnocharis
 rana pipiens
-rat
-rats
 reptile
 reptiles
 ricefish
 ruminant
 ruminants
-salmo gairdneri
-salmon
-serinus canaria
 sheepshead minnow
 sheepshead minnows
-shrew
-shrews
-sorex araneus
 spea multiplicata
+Salmo gairdneri
+salmon
 spotted march frog
 tadpoles
-terrestrial vertrebrates
-toad
 treefrog
-triseriata
+toad
+terrestrial vertrebrates
+Limnodynastes tasmaniensis
 trout
-tshawytscha
-vole
-voles
-vulpes vulpes
-white rabbits
-white-toothed shrew
+Vulpes vulpes
 wistar
-wistar rats
-wood mice
-wood mouse
-wood pigeon
 xenopus laevis
 xenpous leavis
-yellow-necked mouse
 zebra fish
-zebrafish
+zebrafish
+Salmo gairdneri
+minnow
+minnows
+Pimephales promela
+Cyprinodon variegatus
+limnodynastes
+Rana catesbeiana
+R. catesbeiana
+coho salmon
+Oncorhynchus tshawytscha
+O. tshawytscha
+tshawytscha
+catesbeiana
+kisutch
+Pseudacris triseriata
+Pseudacris
+triseriata
+Wood pigeon
+Columba palumbus
+palumbus
+Columbidae
+shrew
+shrews
+bank vole
+common vole
+vole
+voles
+lagomorph
+Wood mouse
+Apodemus sylvaticus
+A. sylvaticus
+Apodemus flavicollis
+Apodemus
+mus musculus
+Microtus arvalis
+Microtus agrestis
+Microtus
+Arvicola terrestris
+Sorex araneus
+Myodes glareolus
+yellow-necked mouse
+house mouse
+Oryctolagus cuniculus
+marten
+martes
+white-toothed shrew
+greater white-toothed shrew
+Lepus europaeus
+brown hare
+European brown hare
+European rabbit
+O. cuniculus
+Crocidura russula
+Chinese Hamster
+Rat
+Rats
+Dog
+Chinese hamsters
+Chinese hamster
+Mouse
+Guinea pig
+Wistar rats
+Rabbit
+mammalian
+Japanese quail
+Microtus subterraneus
+Lepomis macrochirus
+P. promelas
+Cyprinus carpio
+Fish
+Ictalurus punctatus
+Carassius carassius
+Lepomis macrochirus
+Poecilia reticulata
+Lebistes reticulatus
+Lepomis macrochirus
+Leiostomus xanthurus
+Pimephales promelas
+Lepomis macrochirus
+Albino rat
+Hen
+Goat
+Livestock
+Guinea Pigs
+Hamster
+wood mouse
+Rabbits
+Mice
+Rainbow trout
+Canary
+Serinus canaria
+Guinea Pig
+Cow
+Pigs
+Poultry
+Guinea-pigs
+White rabbits
+Birds
+Wood mice
+Mallard

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -49,53 +49,30 @@ rule "5: Do not redact in guideline sections"
 		section.redactNot("address", 5, "Section is a guideline section.");
 	end
 
-rule "6: Redact if must redact entry is found"
+
+rule "6: Redact contact information, if applicant is found"
 	when
-		eval(section.contains("must_redact")==true);
+		eval(section.getText().toLowerCase().contains("applicant") == true);
 	then
-		section.redact("name", 6, "must_redact entry was found.");
-		section.redact("address", 6, "must_redact entry was found.");
+		section.redactLineAfter("Name:", "address", 6, "contact information was found");
+		section.redactBetween("Address:", "Contact", "address", 6, "contact information was found");
+		section.redactLineAfter("Contact point:",  "address", 6, "contact information was found");
+		section.redactLineAfter("Phone:", "address", 6, "contact information was found");
+		section.redactLineAfter("Fax:", "address", 6, "contact information was found");
+		section.redactLineAfter("E-mail:", "address", 6, "contact information was found");
+		section.redactLineAfter("Contact:", "address", 6, "contact information was found");
+		section.redactLineAfter("Telephone number:", "address", 6, "contact information was found");
 	end
 
 
-rule "7: Redact contact information, if applicant is found"
+rule "7: Redact contact information, if 'Producer of the plant protection product' is found"
 	when
-		eval(section.headlineContainsWord("applicant") || section.getText().contains("Applicant"));
+		eval(section.getText().contains("Producer of the plant protection product"));
 	then
-		section.redactLineAfter("Name:", "address", 7, "Applicant information was found");
-		section.redactBetween("Address:", "Contact", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Contact point:",  "address", 7, "Applicant information was found");
-		section.redactLineAfter("Phone:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Fax:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Tel.:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Tel:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("E-mail:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Email:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Contact:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Telephone number:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Fax number:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Telephone:", "address", 7, "Applicant information was found");
-		section.redactBetween("No:", "Fax", "address", 7, "Applicant information was found");
-		section.redactBetween("Contact:", "Tel.:", "address", 7, "Applicant information was found");
-	end
-
-rule "8: Redact contact information, if Producer is found"
-	when
-		eval(section.getText().toLowerCase().contains("producer of the plant protection") || section.getText().toLowerCase().contains("producer of the active substance") || section.getText().contains("Manufacturer of the active substance") || section.getText().contains("Manufacturer:") || section.getText().contains("Producer or producers of the active substance"));
-	then
-		section.redactLineAfter("Name:", "address", 8, "Producer was found");
-		section.redactBetween("Address:", "Contact", "address", 8, "Producer was found");
-		section.redactBetween("Contact:", "Phone", "address", 8, "Producer was found");
-		section.redactBetween("Contact:", "Telephone number:", "address", 8, "Producer was found");
-		section.redactBetween("Address:", "Manufacturing", "address", 8, "Producer was found");
-		section.redactLineAfter("Telephone:", "address", 8, "Producer was found");
-		section.redactLineAfter("Phone:", "address", 8, "Producer was found");
-		section.redactLineAfter("Fax:", "address", 8, "Producer was found");
-		section.redactLineAfter("E-mail:", "address", 8, "Producer was found");
-		section.redactLineAfter("Contact:", "address", 8, "Producer was found");
-		section.redactLineAfter("Fax number:", "address", 8, "Producer was found");
-		section.redactLineAfter("Telephone number:", "address", 8, "Producer was found");
-		section.redactLineAfter("Tel:", "address", 8, "Producer was found");
-		section.redactBetween("No:", "Fax", "address", 8, "Producer was found");
-	end
-
+		section.redactLineAfter("Name:", "address", 7, "Producer of the plant protection product was found");
+		section.redactBetween("Address:", "Contact", "address", 7, "Producer of the plant protection product was found");
+		section.redactBetween("Contact:", "Phone", "address", 7, "Producer of the plant protection product was found");
+		section.redactLineAfter("Phone:", "address", 7, "Producer of the plant protection product was found");
+		section.redactLineAfter("Fax:", "address", 7, "Producer of the plant protection product was found");
+		section.redactLineAfter("E-mail:", "address", 7, "Producer of the plant protection product was found");
+	end