Pull request #20: RED-101

Merge in RED/redaction-service from RED-101 to master

* commit 'c93ca745fc61fc2d7f1a1f474a4e3c464091e70d':
  Normalize header information
  Fix test and suppress checkstyle warnings
  Fix PMD errors
  RED-101: Detect vertebrate study row value
  RED-101: Implement table cell and row redaction
  Fix style

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class Point {
+
+    private float x;
+    private float y;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class Rectangle {
+
+    private Point topLeft;
+    private float width;
+    private float height;
+
+    private int page;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java

@@ -0,0 +1,16 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class RedactionLog {
+
+    private List<RedactionLogEntry> redactionLogEntry;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class RedactionLogEntry {
+
+    private String id;
+    private String type;
+    private String value;
+    private String reason;
+    private boolean redacted;
+    private boolean isHint;
+    private String section;
+    private float[] color;
+    private List<Rectangle> positions = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java

@@ -13,4 +13,6 @@ public class RedactionResult {
 
     private byte[] document;
     private int numberOfPages;
+    private RedactionLog redactionLog;
+
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -23,9 +23,6 @@ public interface RedactionResource {
     @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
     RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
 
-    @PostMapping(value = "/rules", produces = MediaType.APPLICATION_JSON_VALUE)
-    String getRules();
-
     @PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE)
     void updateRules(@RequestBody String rules);
 

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -31,6 +31,16 @@
     </dependencyManagement>
 
     <dependencies>
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>redaction-service-api-v1</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>configuration-service-api-v1</artifactId>
+            <version>1.0.12</version>
+        </dependency>
         <dependency>
             <groupId>org.drools</groupId>
             <artifactId>drools-core</artifactId>
@@ -46,11 +56,6 @@
             <artifactId>jts-core</artifactId>
             <version>1.16.1</version>
         </dependency>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>redaction-service-api-v1</artifactId>
-            <version>${project.version}</version>
-        </dependency>
         <!-- commons -->
         <dependency>
             <groupId>com.iqser.gin4.commons</groupId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -1,45 +1,69 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
-import org.kie.internal.io.ResourceFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
 import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 @Import({DefaultWebMvcConfiguration.class})
+@EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {
 
+    @Autowired
+    private RulesClient rulesClient;
+
+
     public static void main(String[] args) {
+
         SpringApplication.run(Application.class, args);
     }
 
-    private static final String drlFile = "drools/rules.drl";
 
     @Bean
     public KieContainer kieContainer() {
 
-        KieServices kieServices = KieServices.Factory.get();
+        try {
+            KieServices kieServices = KieServices.Factory.get();
 
-        KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-        kieFileSystem.write(ResourceFactory.newClassPathResource(drlFile));
-        KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
-        kieBuilder.buildAll();
-        KieModule kieModule = kieBuilder.getKieModule();
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            RulesResponse rules = rulesClient.getRules();
+            if (StringUtils.isEmpty(rules.getRules())) {
+                throw new RuntimeException("Rules cannot be empty.");
+            }
+            InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
+                    .newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
 
-        return kieServices.newKieContainer(kieModule.getReleaseId());
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        } catch (Exception e) {
+            throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
+        }
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -4,8 +4,8 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 
 import lombok.Data;
@@ -17,10 +17,12 @@ public class Document {
 
     private List<Page> pages = new ArrayList<>();
     private List<Paragraph> paragraphs = new ArrayList<>();
-    private Map<Integer, Set<Entity>> entities = new HashMap<>();
+    private Map<Integer, List<Entity>> entities = new HashMap<>();
     private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
     private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
     private StringFrequencyCounter fontCounter= new StringFrequencyCounter();
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
     private boolean headlines;
+
+    private List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Paragraph.java

@@ -10,17 +10,19 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-
 @Data
 @NoArgsConstructor
 public class Paragraph {
 
     private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
+    private String headline;
+
+
+    public SearchableText getSearchableText() {
 
-    public SearchableText getSearchableText(){
         SearchableText searchableText = new SearchableText();
         pageBlocks.forEach(block -> {
-            if(block instanceof TextBlock){
+            if (block instanceof TextBlock) {
                 searchableText.addAll(((TextBlock) block).getSequences());
             }
         });
@@ -28,14 +30,15 @@ public class Paragraph {
     }
 
 
-    public List<Table> getTables(){
+    public List<Table> getTables() {
+
         List<Table> tables = new ArrayList<>();
         pageBlocks.forEach(block -> {
-            if(block instanceof Table){
+            if (block instanceof Table) {
                 tables.add((Table) block);
             }
         });
         return tables;
     }
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/StringFrequencyCounter.java

@@ -5,43 +5,45 @@ import java.util.Map;
 
 import lombok.Getter;
 
-/**
- *
- */
 public class StringFrequencyCounter {
 
     @Getter
-    Map<String, Integer> countPerValue = new HashMap<>();
+    private final Map<String, Integer> countPerValue = new HashMap<>();
 
-    public void add(String value){
-        if(!countPerValue.containsKey(value)){
+
+    public void add(String value) {
+
+        if (!countPerValue.containsKey(value)) {
             countPerValue.put(value, 1);
         } else {
             countPerValue.put(value, countPerValue.get(value) + 1);
         }
     }
 
-    public void addAll(Map<String, Integer> otherCounter){
-        for(Map.Entry<String, Integer> entry: otherCounter.entrySet()){
-            if(countPerValue.containsKey(entry.getKey())){
-                countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey())+ entry.getValue());
+
+    public void addAll(Map<String, Integer> otherCounter) {
+
+        for (Map.Entry<String, Integer> entry : otherCounter.entrySet()) {
+            if (countPerValue.containsKey(entry.getKey())) {
+                countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
             } else {
                 countPerValue.put(entry.getKey(), entry.getValue());
             }
         }
     }
 
-    public String getMostPopular(){
+
+    public String getMostPopular() {
+
         Map.Entry<String, Integer> mostPopular = null;
-        for(Map.Entry<String, Integer> entry: countPerValue.entrySet()){
-            if(mostPopular == null){
+        for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
+            if (mostPopular == null) {
                 mostPopular = entry;
-            } else if(entry.getValue() > mostPopular.getValue()){
+            } else if (entry.getValue() > mostPopular.getValue()) {
                 mostPopular = entry;
             }
         }
         return mostPopular != null ? mostPopular.getKey() : null;
     }
 
-
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -29,20 +29,16 @@ public class BlockificationService {
         float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
         TextPositionSequence prev = null;
 
-
         for (TextPositionSequence word : textPositions) {
 
             boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
             boolean startFromTop = word.getY1() > maxY + word.getHeight();
 
-            if (prev != null &&
-                    (lineSeparation
-                            || startFromTop
-                            || word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines)
-                            || word.getRotation() == 0 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines)
-                            || word.getRotation() == 90 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines)
-                            || word.getRotation() == 90 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines)
-                    )) {
+            if (prev != null && (lineSeparation || startFromTop || word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word
+                    .getX1(), word.getY1(), verticalRulingLines) || word.getRotation() == 0 && isSplittedByRuling(minX, minY, word
+                    .getX1(), word.getY2(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(maxX, minY, word
+                    .getX1(), word.getY1(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(minX, minY, word
+                    .getX1(), word.getY2(), verticalRulingLines))) {
 
                 TextBlock cb1 = buildTextBlock(chunkWords);
                 chunkBlockList1.add(cb1);
@@ -100,11 +96,12 @@ public class BlockificationService {
             styleFrequencyCounter.add(wordBlock.getFontStyle());
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock
+                        .getRotation());
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(),
-                        spatialEntity.getWidth(), spatialEntity.getHeight());
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
+                        .getHeight());
             }
         }
 
@@ -122,6 +119,7 @@ public class BlockificationService {
 
 
     private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines) {
+
         for (Ruling ruling : rulingLines) {
             if (ruling.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
                 return true;
@@ -133,7 +131,6 @@ public class BlockificationService {
 
     public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
 
-
         float minX = 10000;
         float maxX = -100;
         float minY = 10000;
@@ -147,7 +144,6 @@ public class BlockificationService {
 
             for (AbstractTextContainer container : page.getTextBlocks()) {
 
-
                 if (container instanceof TextBlock) {
                     TextBlock textBlock = (TextBlock) container;
                     if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
@@ -179,16 +175,15 @@ public class BlockificationService {
                     }
                 }
 
-
                 if (container instanceof Table) {
                     Table table = (Table) container;
                     for (List<Cell> row : table.getRows()) {
-                        for (Cell column : row) {
+                        for (Cell cell : row) {
 
-                            if (column == null || column.getTextBlocks() == null) {
+                            if (cell == null || cell.getTextBlocks() == null) {
                                 continue;
                             }
-                            for (TextBlock textBlock : column.getTextBlocks()) {
+                            for (TextBlock textBlock : cell.getTextBlocks()) {
                                 if (textBlock.getMinX() < minX) {
                                     minX = textBlock.getMinX();
                                 }
@@ -211,5 +206,4 @@ public class BlockificationService {
         return new Rectangle(minY, minX, maxX - minX, maxY - minY);
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/DictionaryClient.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
+import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
+
+@FeignClient(name = "DictionaryResource", url = "http://" + RulesResource.SERVICE_NAME + ":8080")
+public interface DictionaryClient extends DictionaryResource {
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/RulesClient.java

@@ -0,0 +1,9 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
+
+@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
+public interface RulesClient extends RulesResource {
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -8,6 +8,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
@@ -24,9 +25,7 @@ import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlatte
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
 
 import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
 
-@Slf4j
 @RestController
 @RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
@@ -38,7 +37,7 @@ public class RedactionController implements RedactionResource {
     private final PdfFlattenService pdfFlattenService;
     private final DroolsExecutionService droolsExecutionService;
 
-
+    @Override
     public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) {
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
@@ -50,17 +49,18 @@ public class RedactionController implements RedactionResource {
 
             if (redactionRequest.isFlatRedaction()) {
                 PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
-                return convert(flatDocument, classifiedDoc.getPages().size());
+                return convert(flatDocument, classifiedDoc.getPages().size(), new RedactionLog(classifiedDoc.getRedactionLogEntities()));
             }
 
-            return convert(pdDocument, classifiedDoc.getPages().size());
+            return convert(pdDocument, classifiedDoc.getPages().size(), new RedactionLog(classifiedDoc.getRedactionLogEntities()));
 
         } catch (IOException e) {
             throw new RedactionException(e);
         }
+
     }
 
-
+    @Override
     public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
@@ -74,9 +74,10 @@ public class RedactionController implements RedactionResource {
         } catch (IOException e) {
             throw new RedactionException(e);
         }
+
     }
 
-
+    @Override
     public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
@@ -90,10 +91,12 @@ public class RedactionController implements RedactionResource {
         } catch (IOException e) {
             throw new RedactionException(e);
         }
+
     }
 
-
+    @Override
     public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
+
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
             pdDocument.setAllSecurityToBeRemoved(true);
 
@@ -114,24 +117,30 @@ public class RedactionController implements RedactionResource {
         } catch (IOException e) {
             throw new RedactionException(e);
         }
+
     }
 
-    public String getRules() {
-        return droolsExecutionService.getRules();
-    }
-
+    @Override
     public void updateRules(@RequestBody String rules) {
         droolsExecutionService.updateRules(rules);
     }
 
+
     private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
+        return convert(document, numberOfPages, null);
+    }
+
+    private RedactionResult convert(PDDocument document, int numberOfPages, RedactionLog redactionLog) throws IOException {
+
         try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
             document.save(byteArrayOutputStream);
             return RedactionResult.builder()
                     .document(byteArrayOutputStream.toByteArray())
                     .numberOfPages(numberOfPages)
+                    .redactionLog(redactionLog)
                     .build();
         }
+
     }
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -39,6 +39,12 @@ public class TextPositionSequence implements CharSequence {
         return text.charAt(0);
     }
 
+    public char charAt(int index, boolean caseInSensitive) {
+        TextPosition textPosition = textPositionAt(index);
+        String text = textPosition.getUnicode();
+        return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
+    }
+
     @Override
     public TextPositionSequence subSequence(int start, int end) {
         return new TextPositionSequence(textPositions.subList(start, end), page);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,14 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-
 import java.util.ArrayList;
 import java.util.List;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 
 @Data
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity {
 
+    @EqualsAndHashCode.Include
     private final String word;
     private final String type;
     private boolean redaction;
@@ -17,18 +19,35 @@ public class Entity {
     private Integer start;
     private Integer end;
 
-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences) {
+    @EqualsAndHashCode.Include
+    private String headline;
+    private int matchedRule;
+
+    @EqualsAndHashCode.Include
+    private int sectionNumber;
+
+
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber) {
+
         this.word = word;
         this.type = type;
         this.redaction = redaction;
         this.redactionReason = redactionReason;
         this.positionSequences = positionSequences;
+        this.headline = headline;
+        this.matchedRule = matchedRule;
+        this.sectionNumber = sectionNumber;
     }
 
-    public Entity(String word, String type, Integer start, Integer end) {
+
+    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {
+
         this.word = word;
         this.type = type;
         this.start = start;
         this.end = end;
+        this.headline = headline;
+        this.sectionNumber = sectionNumber;
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityPositionSequence.java

@@ -6,15 +6,20 @@ import java.util.UUID;
 
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 
+import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;
 
-
 @Data
 @RequiredArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode
 public class EntityPositionSequence {
 
+    @EqualsAndHashCode.Exclude
     private List<TextPositionSequence> sequences = new ArrayList<>();
     private int pageNumber;
     private final UUID id;
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -8,50 +8,70 @@ import java.util.regex.Pattern;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
-@SuppressWarnings("all")
 public class SearchableText {
 
-    private List<TextPositionSequence> sequences = new ArrayList<>();
+    private final List<TextPositionSequence> sequences = new ArrayList<>();
+
 
     public void add(TextPositionSequence textPositionSequence) {
+
         sequences.add(textPositionSequence);
     }
 
+
     public void addAll(List<TextPositionSequence> textPositionSequences) {
+
         sequences.addAll(textPositionSequences);
     }
 
 
-    public List<EntityPositionSequence> getSequences(String searchString) {
+    @SuppressWarnings("checkstyle:ModifiedControlVariable")
+    public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive) {
 
-        char[] searchChars = searchString.replaceAll("\\n", " ").toCharArray();
+        String normalizedSearchString;
+        if (caseInsensitive) {
+            normalizedSearchString = searchString.toLowerCase();
+        } else {
+            normalizedSearchString = searchString;
+        }
+
+        char[] searchChars = normalizedSearchString.replaceAll("\\n", " ").toCharArray();
         int counter = 0;
 
-
         List<TextPositionSequence> crossSequenceParts = new ArrayList<>();
         List<EntityPositionSequence> finalMatches = new ArrayList<>();
         for (int i = 0; i < sequences.size(); i++) {
             TextPositionSequence partMatch = new TextPositionSequence(sequences.get(i).getPage());
             for (int j = 0; j < sequences.get(i).length(); j++) {
 
-                if(i > 0 && j == 0 && sequences.get(i).charAt(0) == ' ' && sequences.get(i - 1).charAt(sequences.get(i - 1).length() - 1) == ' '
-                || j > 0 && sequences.get(i).charAt(j) == ' ' && sequences.get(i).charAt(j - 1) == ' '){
-                    if(j == sequences.get(i).length() -1  && counter != 0 && !partMatch.getTextPositions().isEmpty()){
+                if (i > 0 && j == 0 && sequences.get(i).charAt(0, caseInsensitive) == ' ' && sequences.get(i - 1)
+                        .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && sequences.get(i)
+                        .charAt(j, caseInsensitive) == ' ' && sequences.get(i).charAt(j - 1, caseInsensitive) == ' ') {
+                    if (j == sequences.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
                         crossSequenceParts.add(partMatch);
                     }
                     continue;
                 }
 
-                if(j == 0 && sequences.get(i).charAt(j) != ' ' && i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1) != ' ' && searchChars[counter] == ' '){
+                if (j == 0 && sequences.get(i).charAt(j, caseInsensitive) != ' ' && i != 0 && sequences.get(i - 1)
+                        .charAt(sequences.get(i - 1)
+                                .length() - 1, caseInsensitive) != ' ' && searchChars[counter] == ' ') {
                     counter++;
                 }
 
-                if (sequences.get(i).charAt(j) == searchChars[counter] || counter != 0 && sequences.get(i).charAt(j) == '-') {
+                if (sequences.get(i)
+                        .charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && sequences.get(i)
+                        .charAt(j, caseInsensitive) == '-') {
 
-                    if(counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i).charAt(j - 1)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1))
-                            || j == 0 && i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1) != ' ' && sequences.get(i).charAt(j) != ' ') {
+                    if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i)
+                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1)
+                            .charAt(sequences.get(i - 1)
+                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && sequences.get(i - 1)
+                            .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) != ' ' && sequences.get(i)
+                            .charAt(j, caseInsensitive) != ' ') {
                         partMatch.add(sequences.get(i).textPositionAt(j));
-                        if (!(j == sequences.get(i).length() -1 && sequences.get(i).charAt(j) == '-' && searchChars[counter] != '-')) {
+                        if (!(j == sequences.get(i).length() - 1 && sequences.get(i)
+                                .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                             counter++;
                         }
                     }
@@ -59,10 +79,13 @@ public class SearchableText {
                     if (counter == searchString.length()) {
                         crossSequenceParts.add(partMatch);
 
-                        if(i == sequences.size() - 1 && j == sequences.get(i).length() -1
-                                || j != sequences.get(i).length() -1 && isSeparator(sequences.get(i).charAt(j +1))
-                                || j == sequences.get(i).length() -1 && isSeparator(sequences.get(i + 1).charAt(0))
-                                || j == sequences.get(i).length() -1 && sequences.get(i).charAt(j) != ' ' && sequences.get(i + 1).charAt(0) != ' ') {
+                        if (i == sequences.size() - 1 && j == sequences.get(i).length() - 1 || j != sequences.get(i)
+                                .length() - 1 && isSeparator(sequences.get(i)
+                                .charAt(j + 1, caseInsensitive)) || j == sequences.get(i)
+                                .length() - 1 && isSeparator(sequences.get(i + 1)
+                                .charAt(0, caseInsensitive)) || j == sequences.get(i).length() - 1 && sequences.get(i)
+                                .charAt(j, caseInsensitive) != ' ' && sequences.get(i + 1)
+                                .charAt(0, caseInsensitive) != ' ') {
                             finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
                         }
 
@@ -72,14 +95,14 @@ public class SearchableText {
                     }
                 } else {
                     counter = 0;
-                    if(!crossSequenceParts.isEmpty()){
+                    if (!crossSequenceParts.isEmpty()) {
                         j--;
                     }
                     crossSequenceParts = new ArrayList<>();
                     partMatch = new TextPositionSequence(sequences.get(i).getPage());
                 }
 
-                if(j == sequences.get(i).length() -1  && counter != 0){
+                if (j == sequences.get(i).length() - 1 && counter != 0) {
                     crossSequenceParts.add(partMatch);
                 }
             }
@@ -89,18 +112,18 @@ public class SearchableText {
     }
 
 
-    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts){
+    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {
 
         UUID id = UUID.randomUUID();
         List<EntityPositionSequence> result = new ArrayList<>();
         int currentPage = -1;
         EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
-        for (TextPositionSequence textPositionSequence :crossSequenceParts){
-            if(currentPage == -1){
+        for (TextPositionSequence textPositionSequence : crossSequenceParts) {
+            if (currentPage == -1) {
                 currentPage = textPositionSequence.getPage();
                 entityPositionSequence.setPageNumber(currentPage);
                 entityPositionSequence.getSequences().add(textPositionSequence);
-            } else if(currentPage == textPositionSequence.getPage()){
+            } else if (currentPage == textPositionSequence.getPage()) {
                 entityPositionSequence.getSequences().add(textPositionSequence);
             } else {
                 result.add(entityPositionSequence);
@@ -114,13 +137,14 @@ public class SearchableText {
 
 
     private boolean isSeparator(char c) {
+
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
 
-
     @Override
     public String toString() {
+
         StringBuilder sb = new StringBuilder();
 
         TextPositionSequence previous = null;
@@ -137,10 +161,14 @@ public class SearchableText {
             previous = word;
         }
 
-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll("  ", " ");
+        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
+                .replaceAll("\n", " ")
+                .replaceAll(" {2}", " ");
     }
 
-    public String getAsStringWithLinebreaks(){
+
+    public String getAsStringWithLinebreaks() {
+
         StringBuilder sb = new StringBuilder();
 
         TextPositionSequence previous = null;
@@ -159,4 +187,4 @@ public class SearchableText {
         return sb.append("\n").toString();
     }
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -3,15 +3,19 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import lombok.Builder;
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 
 @Data
+@Slf4j
 @Builder
 public class Section {
 
@@ -23,117 +27,158 @@ public class Section {
     //This does not contain linebreaks and must always be used for correct offsets.
     private String searchText;
 
+    private String headline;
+
+    private int sectionNumber;
+
+    private Map<String, String> tabularData;
+
+
     public boolean contains(String type) {
+
         return entities.stream().anyMatch(entity -> entity.getType().equals(type));
     }
 
-    public void redact(String type, int ruleNumber, String reason){
+
+    public boolean headlineContainsWord(String word) {
+
+        return StringUtils.containsIgnoreCase(headline, word);
+    }
+
+
+    public void redact(String type, int ruleNumber, String reason) {
+
         entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
+            if (entity.getType().equals(type)) {
                 entity.setRedaction(true);
-                entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
             }
         });
     }
 
-    public void redactNot(String type, int ruleNumber, String reason){
+
+    public void redactNot(String type, int ruleNumber, String reason) {
+
         entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
+            if (entity.getType().equals(type)) {
                 entity.setRedaction(false);
-                entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
-            }
-        });
-    }
-
-    public void highlightAll(String type){
-        entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
-                entity.setRedaction(true);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
             }
         });
     }
 
 
-    public void redactLineAfter(String start, String asType, int ruleNumber, String reason){
+    public void redactLineAfter(String start, String asType, int ruleNumber, String reason) {
 
-        String value = StringUtils.substringBetween(text, start, "\n");
+        String[] values = StringUtils.substringsBetween(text, start, "\n");
 
-        if(value != null){
-            Set<Entity> found  = findEntity(value.trim(), asType);
-            entities.addAll(found);
+        if (values != null) {
+            for (String value : values) {
+                if (StringUtils.isNotBlank(value)) {
+                    Set<Entity> found = findEntities(value.trim(), asType);
+                    entities.addAll(found);
+                }
+            }
         }
 
         // TODO No need to iterate
         entities.forEach(entity -> {
-            if(entity.getType().equals(asType)){
+            if (entity.getType().equals(asType)) {
                 entity.setRedaction(true);
-                entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
             }
         });
 
     }
 
 
+    public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason) {
 
-    public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason){
+        String[] values = StringUtils.substringsBetween(searchText, start, stop);
 
-        String value = StringUtils.substringBetween(searchText, start, stop);
-
-        if(value != null){
-            Set<Entity> found  = findEntity(value.trim(), asType);
-            entities.addAll(found);
+        if (values != null) {
+            for (String value : values) {
+                if (StringUtils.isNotBlank(value)) {
+                    Set<Entity> found = findEntities(value.trim(), asType);
+                    entities.addAll(found);
+                }
+            }
         }
 
         // TODO No need to iterate
         entities.forEach(entity -> {
-            if(entity.getType().equals(asType)){
+            if (entity.getType().equals(asType)) {
                 entity.setRedaction(true);
-                entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
             }
         });
     }
 
 
-
-
-    private Set<Entity> findEntity(String value, String asType) {
+    private Set<Entity> findEntities(String value, String asType) {
 
         Set<Entity> found = new HashSet<>();
 
-                int startIndex;
-                int stopIndex = 0;
-                do {
-                    startIndex = searchText.indexOf(value, stopIndex);
-                    stopIndex = startIndex + value.length();
+        int startIndex;
+        int stopIndex = 0;
+        do {
+            startIndex = searchText.indexOf(value, stopIndex);
+            stopIndex = startIndex + value.length();
 
-                    if (startIndex > -1 &&
-                            (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText.charAt(startIndex - 1))) &&
-                            (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
-                        found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex));
-                    }
-                } while (startIndex > -1);
+            if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText
+                    .charAt(startIndex - 1))) && (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
+                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber));
+            }
+        } while (startIndex > -1);
 
-
-        removeEntitiesContainedInLarger(found);
-
-        return found;
+        return removeEntitiesContainedInLarger(found);
     }
 
 
     private boolean isSeparator(char c) {
+
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
-    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
+    public Set<Entity> removeEntitiesContainedInLarger(Set<Entity> entities) {
+
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
-                if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
                     wordsToRemove.add(inner);
                 }
             }
         }
         entities.removeAll(wordsToRemove);
+        return entities;
     }
 
-}
+
+    public void highlightCell(String cellHeader, int ruleNumber) {
+
+        String value = tabularData.get(cellHeader);
+        if (value == null) {
+            log.warn("Could not find any data for {}.", cellHeader);
+        } else {
+            Set<Entity> found = findEntities(value, "must_redact");
+            if (CollectionUtils.isEmpty(found)) {
+                log.warn("Could not identify value {} in row.", value);
+            } else {
+                Entity entity = found.iterator().next();
+                entity.setRedaction(false);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(cellHeader);
+                entities.add(entity);
+            }
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -1,58 +1,97 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import javax.annotation.PostConstruct;
-
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 
+import feign.FeignException;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-@Slf4j
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class DictionaryService {
 
-    public static final String VERTEBRATES_CODE = "VERTEBRATE";
-    public static final String ADDRESS_CODE = "ADDRESS";
-    public static final String NAME_CODE = "NAME";
-    public static final String NO_REDACTION_INDICATOR = "NO_REDACTION_INDICATOR";
+    private final DictionaryClient dictionaryClient;
+
+    private long dictionaryVersion = -1;
 
     @Getter
     private Map<String, Set<String>> dictionary = new HashMap<>();
 
     @Getter
-    private long generation;
+    private Map<String, float[]> entryColors = new HashMap<>();
 
-    @PostConstruct
-    public void init() {
-        loadFromResourceFiles();
-    }
+    @Getter
+    private List<String> hintTypes = new ArrayList<>();
+
+    @Getter
+    private List<String> caseInsensitiveTypes = new ArrayList<>();
+
+    @Getter
+    private float[] defaultColor;
 
 
     public void updateDictionary() {
-        //TODO
+
+        long version = dictionaryClient.getVersion();
+        if (version > dictionaryVersion) {
+            dictionaryVersion = version;
+            updateDictionaryEntry();
+        }
     }
 
 
-    public void loadFromResourceFiles() {
-        dictionary.computeIfAbsent(NAME_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/names.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/vertebrates.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/addresses.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
+    private void updateDictionaryEntry() {
+
+        try {
+            TypeResponse typeResponse = dictionaryClient.getAllTypes();
+            if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
+                entryColors = typeResponse.getTypes()
+                        .stream()
+                        .collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor));
+                hintTypes = typeResponse.getTypes()
+                        .stream()
+                        .filter(TypeResult::isHint)
+                        .map(TypeResult::getType)
+                        .collect(Collectors.toList());
+                caseInsensitiveTypes = typeResponse.getTypes()
+                        .stream()
+                        .filter(TypeResult::isCaseInsensitive)
+                        .map(TypeResult::getType)
+                        .collect(Collectors.toList());
+                dictionary = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, this::convertEntries));
+                defaultColor = dictionaryClient.getDefaultColor().getColor();
+            }
+        } catch (FeignException e) {
+            log.warn("Got some unknown feignException", e);
+            throw e;
+        }
     }
 
 
-    private String cleanDictionaryEntry(String entry) {
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    private Set<String> convertEntries(String s) {
+        if (caseInsensitiveTypes.contains(s)) {
+            return dictionaryClient.getDictionaryForType(s)
+                    .getEntries()
+                    .stream()
+                    .map(String::toLowerCase)
+                    .collect(Collectors.toSet());
+        }
+        return new HashSet<>(dictionaryClient.getDictionaryForType(s).getEntries());
     }
-}
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -4,8 +4,6 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 
-import javax.annotation.PostConstruct;
-
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
@@ -16,30 +14,43 @@ import org.kie.api.runtime.KieSession;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+
+import lombok.RequiredArgsConstructor;
 
 @Service
+@RequiredArgsConstructor
 public class DroolsExecutionService {
 
+    private final RulesClient rulesClient;
+
     @Autowired
     private KieContainer kieContainer;
 
-    private String currentDrlRules;
-
-    @PostConstruct
-    public void init() {
-        currentDrlRules = ResourceLoader.loadAsString("drools/rules.drl");
-    }
+    private long rulesVersion = -1;
 
     public Section executeRules(Section section) {
+
         KieSession kieSession = kieContainer.newKieSession();
         kieSession.setGlobal("section", section);
         kieSession.insert(section);
         kieSession.fireAllRules();
         kieSession.dispose();
+
         return section;
+
+    }
+
+    public void updateRules() {
+
+        long version = rulesClient.getVersion();
+        if (version > rulesVersion) {
+            rulesVersion = version;
+            updateRules(rulesClient.getRules().getRules());
+        }
+
     }
 
     public void updateRules(String drlAsString) {
@@ -56,15 +67,10 @@ public class DroolsExecutionService {
             kieBuilder.buildAll();
             KieModule kieModule = kieBuilder.getKieModule();
             kieContainer.updateToVersion(kieModule.getReleaseId());
-            currentDrlRules = drlAsString;
         } catch (Exception e) {
-            throw new RulesValidationException("Could not update rules", e);
+            throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
         }
 
     }
 
-    public String getRules() {
-        return currentDrlRules;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -1,18 +1,22 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -31,108 +35,170 @@ public class EntityRedactionService {
     public void processDocument(Document classifiedDoc) {
 
         dictionaryService.updateDictionary();
+        droolsExecutionService.updateRules();
 
         Set<Entity> documentEntities = new HashSet<>();
+        int sectionNumber = 1;
         for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
 
             SearchableText searchableText = paragraph.getSearchableText();
 
             List<Table> tables = paragraph.getTables();
 
-            List<SearchableText> searchableRows = new ArrayList<>();
             for (Table table : tables) {
+                List<String> metadata = table.getHeaders();
                 for (List<Cell> row : table.getRows()) {
                     SearchableText searchableRow = new SearchableText();
-                    for (Cell column : row) {
-                        if (column == null || column.getTextBlocks() == null) {
+                    List<String> cellValues = new ArrayList<>();
+                    for (Cell cell : row) {
+                        if (cell == null || CollectionUtils.isEmpty(cell.getTextBlocks())) {
+                            cellValues.add(null);
                             continue;
                         }
-                        for (TextBlock textBlock : column.getTextBlocks()) {
+                        cellValues.add(cell.getTextBlocks().get(0).getText());
+                        for (TextBlock textBlock : cell.getTextBlocks()) {
                             searchableRow.addAll(textBlock.getSequences());
                         }
                     }
-                    searchableRows.add(searchableRow);
+                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber);
+
+                    Map<String, String> tabularData = toMap(metadata, cellValues);
+                    Section analysedRowSection = droolsExecutionService.executeRules(Section.builder()
+                            .entities(rowEntities)
+                            .text(searchableRow.getAsStringWithLinebreaks())
+                            .searchText(searchableRow.toString())
+                            .headline(table.getHeadline())
+                            .sectionNumber(sectionNumber)
+                            .tabularData(tabularData)
+                            .build());
+
+                    documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), searchableRow));
+                    sectionNumber++;
                 }
+                sectionNumber++;
             }
 
-            Set<Entity> entities = findEntities(searchableText);
-            Section analysedSection = droolsExecutionService.executeRules(Section
-                    .builder()
+            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber);
+            Section analysedSection = droolsExecutionService.executeRules(Section.builder()
                     .entities(entities)
                     .text(searchableText.getAsStringWithLinebreaks())
                     .searchText(searchableText.toString())
+                    .headline(paragraph.getHeadline())
+                    .sectionNumber(sectionNumber)
                     .build());
 
-            for (Entity entity : analysedSection.getEntities()) {
-                entity.setPositionSequences(searchableText.getSequences(entity.getWord()));
+            documentEntities.addAll(clearAndFindPositions(analysedSection.getEntities(), searchableText));
+            sectionNumber++;
+        }
+
+        for (Entity entity : documentEntities) {
+            Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
+            for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
+                sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
+                        .add(entityPositionSequence);
             }
 
-            documentEntities.addAll(analysedSection.getEntities());
-
-            for (SearchableText searchableRow : searchableRows) {
-                Set<Entity> rowEntities = findEntities(searchableRow);
-
-                Section analysedRowSection = droolsExecutionService.executeRules(Section
-                        .builder()
-                        .entities(rowEntities)
-                        .text(searchableRow.getAsStringWithLinebreaks())
-                        .searchText(searchableRow.toString())
-                        .build());
-
-                for (Entity entity : analysedRowSection.getEntities()) {
-                    entity.setPositionSequences(searchableRow.getSequences(entity.getWord()));
-                }
-                documentEntities.addAll(analysedRowSection.getEntities());
+            for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
+                classifiedDoc.getEntities()
+                        .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
+                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(),
+                                entity.getRedactionReason(), entry
+                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber()));
             }
         }
 
-        documentEntities.forEach(entity -> {
-            entity.getPositionSequences().forEach(sequence -> {
-                classifiedDoc.getEntities().computeIfAbsent(sequence.getPageNumber(), (x) -> new HashSet<>()).add(
-                        new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List.of(sequence))
-                );
-            });
-        });
     }
 
 
-    private Set<Entity> findEntities(SearchableText searchableText) {
+    private Map<String, String> toMap(List<String> keys, List<String> values) {
 
-        String normalizedInputString = searchableText.toString();
+        if (keys.size() != values.size()) {
+            throw new RuntimeException("Cannot merge lists of unequal size.");
+        }
+        Map<String, String> result = new HashMap<>();
+        for (int i = 0; i < keys.size(); i++) {
+            result.put(keys.get(i), values.get(i));
+        }
+
+        return result;
+
+    }
+
+
+    private Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text) {
+
+        removeEntitiesContainedInLarger(entities);
+
+        for (Entity entity : entities) {
+            if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
+                entity.setPositionSequences(text.getSequences(entity.getWord(), true));
+            } else {
+                entity.setPositionSequences(text.getSequences(entity.getWord(), false));
+            }
+        }
+
+        return entities;
+    }
+
+
+    private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber) {
 
         Set<Entity> found = new HashSet<>();
-        for (Map.Entry<String, Set<String>> entry : dictionaryService.getDictionary().entrySet()) {
-            for (String value : entry.getValue()) {
-                int startIndex;
-                int stopIndex = 0;
-                do {
-                    startIndex = normalizedInputString.indexOf(value, stopIndex);
-                    stopIndex = startIndex + value.length();
+        if (StringUtils.isEmpty(searchableText.toString()) && StringUtils.isEmpty(headline)) {
+            return found;
+        }
 
-                    if (startIndex > -1 &&
-                            (startIndex == 0 || Character.isWhitespace(normalizedInputString.charAt(startIndex - 1)) || isSeparator(normalizedInputString.charAt(startIndex - 1))) &&
-                            (stopIndex == normalizedInputString.length() || isSeparator(normalizedInputString.charAt(stopIndex)))) {
-                        found.add(new Entity(normalizedInputString.substring(startIndex, stopIndex), entry.getKey(), startIndex, stopIndex));
-                    }
-                } while (startIndex > -1);
+        String inputString = searchableText.toString();
+        String lowercaseInputString = inputString.toLowerCase();
+        for (Map.Entry<String, Set<String>> entry : dictionaryService.getDictionary().entrySet()) {
+            if (dictionaryService.getCaseInsensitiveTypes().contains(entry.getKey())) {
+                found.addAll(find(lowercaseInputString, entry.getValue(), entry.getKey(), headline, sectionNumber));
+            } else {
+                found.addAll(find(inputString, entry.getValue(), entry.getKey(), headline, sectionNumber));
             }
         }
 
         removeEntitiesContainedInLarger(found);
 
         return found;
+
     }
 
+
+    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber) {
+
+        Set<Entity> found = new HashSet<>();
+        for (String value : values) {
+            int startIndex;
+            int stopIndex = 0;
+            do {
+                startIndex = inputString.indexOf(value, stopIndex);
+                stopIndex = startIndex + value.length();
+
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex,
+                            headline, sectionNumber));
+                }
+            } while (startIndex > -1);
+        }
+        return found;
+    }
+
+
     private boolean isSeparator(char c) {
+
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
 
     public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
-                if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
                     wordsToRemove.add(inner);
                 }
             }
@@ -140,5 +206,4 @@ public class EntityRedactionService {
         entities.removeAll(wordsToRemove);
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
 import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@@ -20,38 +19,12 @@ public class ResourceLoader {
         if (resource == null) {
             throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
         }
-        try (InputStream is = resource.openStream();
-             InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
-             BufferedReader br = new BufferedReader(isr)) {
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
             return br.lines().collect(Collectors.toSet());
         } catch (IOException e) {
             throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
         }
+
     }
 
-
-    public String loadAsString(String classpathPath) {
-        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
-        }
-        try (InputStream is = resource.openStream();
-             InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
-             BufferedReader br = new BufferedReader(isr)) {
-            StringBuffer sb = new StringBuffer();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
-        }
-    }
-}
-
-
-
-
-
-
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -29,7 +29,6 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@SuppressWarnings("PMD")
 public class PdfSegmentationService {
 
     private final RulingCleaningService rulingCleaningService;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -4,6 +4,8 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
@@ -14,50 +16,52 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
 @Service
-@SuppressWarnings("all")
 public class SectionsBuilderService {
 
-
     public void buildSections(Document document) {
 
         List<AbstractTextContainer> chunkWords = new ArrayList<>();
-        List<Paragraph> chunkBlockList1 = new ArrayList<>();
+        List<Paragraph> chunkBlockList = new ArrayList<>();
 
         AbstractTextContainer prev = null;
 
+        String lastHeadline = "";
+        Table previousTable = null;
         for (Page page : document.getPages()) {
             for (AbstractTextContainer current : page.getTextBlocks()) {
 
-                if (current.getClassification() == null || current.getClassification().equals("Header") || current.getClassification().equals("Footer")) {
+                if (current.getClassification() == null || current.getClassification()
+                        .equals("Header") || current.getClassification().equals("Footer")) {
                     continue;
                 }
 
                 current.setPage(page.getPageNumber());
 
                 if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
-
-                    Paragraph cb1 = buildTextBlock(chunkWords);
-                    chunkBlockList1.add(cb1);
+                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline, previousTable);
+                    chunkBlock.setHeadline(lastHeadline);
+                    lastHeadline = current.getText();
+                    if (CollectionUtils.isNotEmpty(chunkBlock.getTables())) {
+                        previousTable = chunkBlock.getTables().get(0);
+                    }
+                    chunkBlockList.add(chunkBlock);
                     chunkWords = new ArrayList<>();
-
                 }
 
                 chunkWords.add(current);
-
                 prev = current;
             }
         }
 
-        Paragraph cb1 = buildTextBlock(chunkWords);
-        if (cb1 != null) {
-            chunkBlockList1.add(cb1);
-        }
+        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline, previousTable);
+        chunkBlock.setHeadline(lastHeadline);
+        chunkBlockList.add(chunkBlock);
 
-        document.setParagraphs(chunkBlockList1);
+        document.setParagraphs(chunkBlockList);
     }
 
 
-    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList) {
+    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline, Table previousTable) {
 
         Paragraph paragraph = new Paragraph();
         TextBlock textBlock = null;
@@ -66,43 +70,60 @@ public class SectionsBuilderService {
         boolean splitByTable = false;
 
         Iterator<AbstractTextContainer> itty = wordBlockList.iterator();
-        boolean alreadyAdded= false;
+        boolean alreadyAdded = false;
+        AbstractTextContainer previous = null;
         while (itty.hasNext()) {
             AbstractTextContainer container = itty.next();
 
             if (container instanceof Table) {
+                Table table = (Table) container;
                 splitByTable = true;
 
+                if (previous != null && previous.getText().startsWith("Table ")) {
+                    table.setHeadline(previous.getText());
+                } else {
+                    table.setHeadline("Table in: " + lastHeadline);
+                }
+                // Distribute header information for subsequent tables
+                if (previousTable != null && hasInvalidHeaderInformation(table) && hasValidHeaderInformation(previousTable) &&
+                        (previousTable.isVerticalHeader() && previousTable.getRowCount() == table.getRowCount() ||
+                                previousTable.getColCount() == table.getColCount())) {
+                    table.setHeaders(previousTable.getHeaders());
+                }
 
                 if (textBlock != null && !alreadyAdded) {
                     paragraph.getPageBlocks().add(textBlock);
-                    alreadyAdded =true;
+                    alreadyAdded = true;
                 }
-                paragraph.getPageBlocks().add(container);
+                paragraph.getPageBlocks().add(table);
                 continue;
             }
 
             TextBlock wordBlock = (TextBlock) container;
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
             } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
                 alreadyAdded = false;
             } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
                 textBlock.setPage(pageBefore);
                 paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(),
-                        spatialEntity.getWidth(), spatialEntity.getHeight());
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
+                        .getHeight());
             }
             pageBefore = wordBlock.getPage();
             splitByTable = false;
+            previous = container;
         }
 
         if (textBlock != null && !alreadyAdded) {
@@ -112,4 +133,23 @@ public class SectionsBuilderService {
     }
 
 
-}
+    private boolean hasValidHeaderInformation(Table table) {
+
+        return !hasInvalidHeaderInformation(table);
+    }
+
+
+    private boolean hasInvalidHeaderInformation(Table table) {
+
+        if (CollectionUtils.isEmpty(table.getHeaders())) {
+            return true;
+        }
+        if (table.getHeaders().stream().anyMatch(StringUtils::isEmpty)) {
+            return true;
+        }
+
+        return false;
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java

@@ -16,11 +16,17 @@ public class Cell extends Rectangle {
 
     private List<TextBlock> textBlocks = new ArrayList<>();
 
+
     public Cell(Point2D topLeft, Point2D bottomRight) {
-        super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()));
+
+        super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight
+                .getY() - topLeft.getY()));
     }
 
+
     public void addTextBlock(TextBlock textBlock) {
+
         textBlocks.add(textBlock);
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/RectangleSpatialIndex.java

@@ -8,25 +8,28 @@ import org.locationtech.jts.index.strtree.STRtree;
 
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
 
-
 @SuppressWarnings("all")
 public class RectangleSpatialIndex<T extends Rectangle> {
-    
 
     private final STRtree si = new STRtree();
     private final List<T> rectangles = new ArrayList<>();
 
+
     public void add(T te) {
+
         rectangles.add(te);
         si.insert(new Envelope(te.getLeft(), te.getRight(), te.getBottom(), te.getTop()), te);
     }
-    
-    public List<T> contains(Rectangle r) {
-        List<T> intersection = si.query(new Envelope(r.getLeft(), r.getRight(), r.getTop(), r.getBottom()));
+
+
+    public List<T> contains(Rectangle rectangle) {
+
+        List<T> intersection = si.query(new Envelope(rectangle.getLeft(), rectangle.getRight(), rectangle.getTop(), rectangle
+                .getBottom()));
         List<T> rv = new ArrayList<T>();
 
-        for (T ir: intersection) {
-            if (r.contains(ir)) {
+        for (T ir : intersection) {
+            if (rectangle.contains(ir)) {
                 rv.add(ir);
             }
         }
@@ -34,18 +37,22 @@ public class RectangleSpatialIndex<T extends Rectangle> {
         Utils.sort(rv, Rectangle.ILL_DEFINED_ORDER);
         return rv;
     }
-    
+
+
     public List<T> intersects(Rectangle r) {
+
         List rv = si.query(new Envelope(r.getLeft(), r.getRight(), r.getTop(), r.getBottom()));
         return rv;
     }
-    
+
+
     /**
      * Minimum bounding box of all the Rectangles contained on this RectangleSpatialIndex
-     * 
+     *
      * @return a Rectangle
      */
     public Rectangle getBounds() {
+
         return Rectangle.boundingBoxOf(rectangles);
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java

@@ -8,27 +8,45 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections4.CollectionUtils;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
 
 import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
 
-@SuppressWarnings("all")
+@Slf4j
 public class Table extends AbstractTextContainer {
 
     private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();
 
-    private RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();
+    private final RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();
 
     @Getter
-    private int rowCount = 0;
+    @Setter
+    private String headline;
+
     @Getter
-    private int colCount = 0;
+    private int rowCount;
 
-    private int rotation = 0;
+    @Getter
+    private int colCount;
 
-    private List<List<Cell>> memoizedRows = null;
+    private final int rotation;
+
+    private List<List<Cell>> rows;
+
+    @Getter
+    @Setter
+    private List<String> headers;
+
+    @Getter
+    private boolean verticalHeader;
 
     public Table(List<Cell> cells, Rectangle area, int rotation) {
 
@@ -42,16 +60,87 @@ public class Table extends AbstractTextContainer {
 
     }
 
+
     public List<List<Cell>> getRows() {
 
-        if (memoizedRows == null) {
-            memoizedRows = computeRows();
+        if (rows == null) {
+            rows = computeRows();
+            headers = computeHeaders();
         }
 
-        return memoizedRows;
+        return rows;
 
     }
 
+
+    /**
+     * Detect header cells (either first row or first column):
+     * Column is marked as header if cell text is bold and row cell text is not bold.
+     * Defaults to row.
+     */
+    private List<String> computeHeaders() {
+
+        boolean allBold = true;
+        List<Cell> rowCells = rows.get(0);
+        for (Cell cell : rowCells) {
+            if (cell == null || CollectionUtils.isEmpty(cell.getTextBlocks()) ||
+                    !cell.getTextBlocks().get(0).getMostPopularWordStyle().equals("bold")) {
+                allBold = false;
+                break;
+            }
+        }
+        if (!allBold) {
+            allBold = true;
+            List<Cell> firstColCells = new ArrayList<>();
+            for (List<Cell> row : rows) {
+                Cell firstInRow = row.get(0);
+                if (firstInRow == null || CollectionUtils.isEmpty(firstInRow.getTextBlocks()) ||
+                        !firstInRow.getTextBlocks().get(0).getMostPopularWordStyle().equals("bold")) {
+                    allBold = false;
+                    break;
+                }
+                firstColCells.add(firstInRow);
+            }
+            if (allBold) {
+                log.info("Headers are in first column");
+                verticalHeader = true;
+                return firstColCells.stream().map(cell -> {
+                    if (CollectionUtils.isNotEmpty(cell.getTextBlocks())) {
+                        return TextNormalizationUtilities.removeHyphenLineBreaks(cell.getTextBlocks().get(0).getText())
+                                .replaceAll("\n", " ")
+                                .replaceAll(" ", " ");
+                    } else {
+                        return null;
+                    }
+                }).collect(Collectors.toList());
+            } else {
+                log.info("Headers are defaulted in first row.");
+                return rowCells.stream().map(cell -> {
+                    if (cell != null && CollectionUtils.isNotEmpty(cell.getTextBlocks())) {
+                        return TextNormalizationUtilities.removeHyphenLineBreaks(cell.getTextBlocks().get(0).getText())
+                                .replaceAll("\n", " ")
+                                .replaceAll(" ", " ");
+                    } else {
+                        return null;
+                    }
+                }).collect(Collectors.toList());
+            }
+        } else {
+            log.info("Headers are in first row.");
+            return rowCells.stream().map(cell -> {
+                if (CollectionUtils.isNotEmpty(cell.getTextBlocks())) {
+                    return TextNormalizationUtilities.removeHyphenLineBreaks(cell.getTextBlocks().get(0).getText())
+                            .replaceAll("\n", " ")
+                            .replaceAll(" ", " ");
+                } else {
+                    return null;
+                }
+            }).collect(Collectors.toList());
+        }
+
+    }
+
+
     private List<List<Cell>> computeRows() {
 
         List<List<Cell>> rows = new ArrayList<>();
@@ -88,7 +177,8 @@ public class Table extends AbstractTextContainer {
 
     }
 
-    public void add(Cell chunk, int row, int col) {
+
+    private void add(Cell chunk, int row, int col) {
 
         rowCount = Math.max(rowCount, row + 1);
         colCount = Math.max(colCount, col + 1);
@@ -98,6 +188,7 @@ public class Table extends AbstractTextContainer {
 
     }
 
+
     private void addCells(List<Cell> cells) {
 
         if (cells.isEmpty()) {
@@ -126,14 +217,9 @@ public class Table extends AbstractTextContainer {
             while (rowCells.hasNext()) {
                 Cell cell = rowCells.next();
                 if (i > 0) {
-                    List<List<Cell>> others = rowsOfCells(
-                            si.contains(
-                                    new Rectangle(cell.getBottom(),
-                                            si.getBounds().getLeft(),
-                                            cell.getLeft() - si.getBounds().getLeft() + 1,
-                                            si.getBounds().getBottom() - cell.getBottom()
-                                    )
-                            ));
+                    List<List<Cell>> others = rowsOfCells(si.contains(new Rectangle(cell.getBottom(), si.getBounds()
+                            .getLeft(), cell.getLeft() - si.getBounds().getLeft() + 1, si.getBounds().getBottom() - cell
+                            .getBottom())));
 
                     for (List<Cell> r : others) {
                         jumpToColumn = Math.max(jumpToColumn, r.size());
@@ -153,7 +239,9 @@ public class Table extends AbstractTextContainer {
         }
     }
 
+
     private static List<List<Cell>> rowsOfCells(List<Cell> cells) {
+
         Cell c;
         float lastTop;
         List<List<Cell>> rv = new ArrayList<>();
@@ -163,19 +251,10 @@ public class Table extends AbstractTextContainer {
             return rv;
         }
 
-        Collections.sort(cells, new Comparator<Cell>() {
-            @Override
-            public int compare(Cell arg0, Cell arg1) {
-                return Double.compare(arg0.getLeft(), arg1.getLeft());
-            }
-        });
+        cells.sort(Comparator.comparingDouble(Rectangle::getLeft));
 
-        Collections.sort(cells, Collections.reverseOrder(new Comparator<Cell>() {
-            @Override
-            public int compare(Cell arg0, Cell arg1) {
-                return Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1.getBottom(),2));
-            }
-        }));
+        cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1
+                .getBottom(), 2))));
 
         Iterator<Cell> iter = cells.iterator();
         c = iter.next();
@@ -196,6 +275,7 @@ public class Table extends AbstractTextContainer {
         return rv;
     }
 
+
     @Override
     public String getText() {
 
@@ -232,6 +312,7 @@ public class Table extends AbstractTextContainer {
         return sb.toString();
     }
 
+
     public String getTextAsHtml() {
 
         StringBuilder sb = new StringBuilder();
@@ -265,22 +346,30 @@ public class Table extends AbstractTextContainer {
         return sb.toString();
     }
 
-    class CellPosition implements Comparable<CellPosition> {
+
+    static class CellPosition implements Comparable<CellPosition> {
 
         CellPosition(int row, int col) {
+
             this.row = row;
             this.col = col;
         }
 
-        final int row, col;
+
+        final int row;
+        final int col;
+
 
         @Override
         public int hashCode() {
+
             return row + 101 * col;
         }
 
+
         @Override
         public boolean equals(Object obj) {
+
             if (this == obj) {
                 return true;
             }
@@ -294,10 +383,12 @@ public class Table extends AbstractTextContainer {
             return row == other.row && col == other.col;
         }
 
+
         @Override
         public int compareTo(CellPosition other) {
-            int rowdiff = row - other.row;
-            return rowdiff != 0 ? rowdiff : col - other.col;
+
+            int rowDiff = row - other.row;
+            return rowDiff != 0 ? rowDiff : col - other.col;
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java

@@ -1,14 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.visualization.service;
 
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.ADDRESS_CODE;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NAME_CODE;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NO_REDACTION_INDICATOR;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.VERTEBRATES_CODE;
-
 import java.awt.Color;
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
@@ -20,12 +16,16 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -38,6 +38,8 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class AnnotationHighlightService {
 
+    private final DictionaryService dictionaryService;
+
 
     public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException {
 
@@ -77,6 +79,8 @@ public class AnnotationHighlightService {
 
             for (Entity entity : classifiedDoc.getEntities().get(page)) {
 
+                RedactionLogEntry redactionLogEntry = new RedactionLogEntry();
+
                 for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
 
                     if (flatRedaction && !isRedactionType(entity)) {
@@ -91,47 +95,54 @@ public class AnnotationHighlightService {
                         float posXEnd;
                         float posYInit;
                         float posYEnd;
-                        float[] quadPoints;
 
                         if (textPositions.getTextPositions().get(0).getRotation() == 90) {
 
                             posXEnd = textPositions.getTextPositions().get(0).getYDirAdj() + 2;
                             posXInit = textPositions.getTextPositions().get(0).getYDirAdj() - height;
                             posYInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posYEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() - height + 2;
-
-                            quadPoints = new float[]{posXInit, posYInit, posXInit, posYEnd + height + 2, posXEnd, posYInit, posXEnd, posYEnd + height + 2};
+                            posYEnd = textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getXDirAdj() - height + 4;
                         } else {
 
                             posXInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posXEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() + textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getWidth() + 1;
-                            posYInit = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj();
-                            posYEnd = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getYDirAdj();
-                            quadPoints = new float[]{posXInit, posYEnd + height + 2, posXEnd, posYEnd + height + 2, posXInit, posYInit - 2, posXEnd, posYEnd - 2};
+                            posXEnd = textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getXDirAdj() + textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getWidth() + 1;
+                            posYInit = textPositions.getTextPositions()
+                                    .get(0)
+                                    .getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj() - 2;
+                            posYEnd = textPositions.getTextPositions()
+                                    .get(0)
+                                    .getPageHeight() - textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getYDirAdj() + 2;
                         }
 
+                        Rectangle textHighlightRectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
 
                         List<PDAnnotation> annotations = pdPage.getAnnotations();
                         PDAnnotationTextMarkup highlight = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
                         highlight.constructAppearances();
 
-                        PDRectangle position = new PDRectangle();
-                        position.setLowerLeftX(posXInit);
-                        position.setLowerLeftY(posYEnd);
-                        position.setUpperRightX(posXEnd);
-                        position.setUpperRightY(posYEnd + height);
+                        PDRectangle annotationPosition = new PDRectangle();
+                        annotationPosition.setLowerLeftX(posXInit);
+                        annotationPosition.setLowerLeftY(posYEnd);
+                        annotationPosition.setUpperRightX(posXEnd);
+                        annotationPosition.setUpperRightY(posYEnd + height);
 
-                        highlight.setRectangle(position);
-                        if (!flatRedaction) {
+                        highlight.setRectangle(annotationPosition);
+                        if (!flatRedaction && !isHint(entity)) {
                             highlight.setAnnotationName(entityPositionSequence.getId().toString());
                             highlight.setTitlePopup(entityPositionSequence.getId().toString());
-                            highlight.setContents(entity.getRedactionReason());
+                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity
+                                    .getHeadline() + "\"");
                         }
 
-                        // quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
-                        // of the area to be highlighted
-
-                        highlight.setQuadPoints(quadPoints);
+                        highlight.setQuadPoints(toQuadPoints(textHighlightRectangle));
 
                         PDColor color;
                         if (flatRedaction) {
@@ -142,47 +153,69 @@ public class AnnotationHighlightService {
 
                         highlight.setColor(color);
                         annotations.add(highlight);
+
+                        redactionLogEntry.getPositions().add(textHighlightRectangle);
+
                     }
+                    redactionLogEntry.setId(entityPositionSequence.getId().toString());
                 }
+                redactionLogEntry.setColor(getColor(entity));
+                redactionLogEntry.setReason(entity.getRedactionReason());
+                redactionLogEntry.setValue(entity.getWord());
+                redactionLogEntry.setType(entity.getType());
+                redactionLogEntry.setRedacted(entity.isRedaction());
+                redactionLogEntry.setSection(entity.getHeadline());
+                redactionLogEntry.setHint(isHint(entity));
+                classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
             }
 
         }
     }
 
 
+    private float[] toQuadPoints(Rectangle rectangle) {
+
+        // quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
+        // of the area to be highlighted
+        return new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getTopLeft()
+                .getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rectangle.getTopLeft().getX(), rectangle.getTopLeft()
+                .getY() + rectangle.getHeight(), rectangle.getTopLeft()
+                .getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
+    }
+
+
     private boolean isRedactionType(Entity entity) {
+
         if (!entity.isRedaction()) {
             return false;
         }
-        if (entity.getType().equals(ADDRESS_CODE)) {
-            return true;
+        if (isHint(entity)) {
+            return false;
         }
-        if (entity.getType().equals(NAME_CODE)) {
-            return true;
-        }
-        return false;
+        return true;
     }
 
 
     private float[] getColor(Entity entity) {
-        if (!entity.isRedaction()) {
+
+        if (!entity.isRedaction() && !isHint(entity)) {
             return new float[]{0.627f, 0.627f, 0.627f};
         }
-        if (entity.getType().equals(VERTEBRATES_CODE)) {
-            return new float[]{0, 1, 0};
+
+        if (!dictionaryService.getEntryColors().containsKey(entity.getType())) {
+            return dictionaryService.getDefaultColor();
         }
-        if (entity.getType().equals(ADDRESS_CODE)) {
-            return new float[]{0, 1, 1};
-        }
-        if (entity.getType().equals(NAME_CODE)) {
-            return new float[]{1, 1, 0};
-        }
-        if (entity.getType().equals(NO_REDACTION_INDICATOR)) {
-            return new float[]{1, 0.502f, 0};
-        }
-        return null;
+
+        return dictionaryService.getEntryColors().get(entity.getType());
     }
 
+    private boolean isHint(Entity entity) {
+        List<String> hintTypes = dictionaryService.getHintTypes();
+        if (CollectionUtils.isNotEmpty(hintTypes) && hintTypes.contains(entity.getType())) {
+            return true;
+        }
+        return false;
+    }
 
     private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
 
@@ -208,13 +241,15 @@ public class AnnotationHighlightService {
 
 
     private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
+
         for (List<Cell> row : table.getRows()) {
             for (Cell cell : row) {
 
                 if (cell != null) {
                     contentStream.setLineWidth(0.5f);
                     contentStream.setStrokingColor(Color.CYAN);
-                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell.getHeight());
+                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell
+                            .getHeight());
                     contentStream.stroke();
 
 //                    contentStream.setStrokingColor(Color.GREEN);
@@ -239,4 +274,5 @@ public class AnnotationHighlightService {
             contentStream.endText();
         }
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/resources/dictionaries/NoRedactionIndicator.txt

@@ -1,3 +0,0 @@
-In Vitro
-In vitro
-in vitro

redaction-service-v1/redaction-service-server-v1/src/main/resources/dictionaries/addresses.txt

@@ -1,796 +0,0 @@
-Aquatic BioSystems Inc, Fort Collins, Colorado, USA
-Aquatic BioSystems, Inc., Ft. Collins, Colorado, USA.
-Biological Research Laboratory (BRL), Füllinsdorf, Switzerland.
-Biological Serviced Section, Alderley Park, Macclesfield, Cheshire
-Harlan Laboratories Ltd., Itingen,
-Jealott’s Hill, International Research Station, Bracknell,
-Jealott’s Hill, International Research Station, Bracknell, RG42 6EY, United Kingdom
-Jealott’s Hill, International Research Station, Bracknell, RG42 6EY, United Kingdom.
-Obtained from P. Hohler, trout breeding station Zeiningen, CH-4314 Zeiningen, Switzerland
-P. Hohler, Forellenzucht Zeiningen, CH-4314 Zeiningen Switzerland
-P.Hohler trout breeding station Zeiningen, CH-4314 Zeiningen, Swit-zerland, and held in the test facility for more than 2 weeks
-RCC Biotechnology & Animal Breeding Division, Füllinsdorf,
-RCC Biotechnology & Animal Breeding Division, Füllinsdorf, Switzerland
-Sequani Limited, Ledbury, United Kingdom, BFI0274
-Springborn Laboratories Inc., 790 Main St., Wareham, Massachusetts, 02571-1075, USA.
-Syngenta, Jealott’s Hill, International Research Station, Bracknell, RG42 6EY, United Kingdom
-adama max rudong 2014 - huifeng
-animal metabolism, dietary exposure, product safety, research and development, ciba-geigy limited, basle, switzerland
-aquatic bio systems, inc., fort collins, colorado.
-aquatic bioassay laboratory, baton rouge, louisiana
-arysta lifescience north america, llc, cary, nc, usa
-arysta lifescience sas, noguères, france
-bayer crop-science
-bayer crop-science ag
-bc potter, rosedean, woodhurst, cambridgeshire, england
-biospheric inc., rockville, usa
-birds obtained from m & m quail farm, 4090 campbell road, gillsville, ga 30543 u.s.a
-brixham environmental laboratory, astrazeneca uk limited, brixham, uk
-brixham environmental laboratory, brixham, uk
-brixham environmental laboratory, brixham, united kingdom
-brood stock maintained at springborn laboratories
-buffalo creek quail farm, po box 579, ellerbe, nc
-bybrook bass hatchery, connecticut
-c.i.t, miserey, france
-celsius property b.v., amsterdam, netherlands
-central toxicology laboratory
-central toxicology laboratory (ctl), cheshire, united kingdom
-central toxicology laboratory (ctl), cheshire, united kingdom, hr2464
-central toxicology laboratory, alderley park, macclesfield, cheshire uk
-centre international de toxicologie (c.i.t.), miserey, 27005 evreux, france
-charles river
-charles river (uk) limited
-charles river (uk) limited, margate, kent, ct9 4lt, england.
-charles river aquaria, margate, uk
-charles river breeding laboratories, raleigh, nc, usa
-charles river deutschland gmbh, stolzenseeweg 32-36, d-88353 kisslegg / germany
-charles river france
-charles river laboratories edinburgh ltd, tranent, eh33 2ne
-charles river laboratories edinburgh ltd, tranent, eh33 2ne, uk
-charles river laboratories france, bp 0109, f-69592 l’arbresle
-charles river laboratories, edinburgh, united kingdom
-charles river laboratories, edinburgh, united kingdom, 38674
-charles river laboratories, portage, mi
-charles river laboratories, raleigh, nc, usa
-charles river uk limited, margate, kent.
-charles river, 76410, saint-aubin-les-elbeuf, france
-cheshire, united kingdom,
-china agricultural university, no.2, yuan ming yuan west road, haidian district, beijing, 100193, p.r. china
-ciba-geigy agricultural division, 410 swing road, p.o. box 18300, greensboro, north carolina 27419
-ciba-geigy basel, oekotoxikologie, basel, switzerland, 953609
-ciba-geigy corp. environmental health centre, farmington, ct, usa.
-ciba-geigy corp., greensboro, us
-ciba-geigy corp., vero beach, us
-ciba-geigy corporation agricultural division, environmental health centre (ehc), 400 farmington avenue, farmington, ct 06032
-ciba-geigy limited, animal production unit, basle, switzerland.
-ciba-geigy limited, animal production unit, stein, switzerland.
-ciba-geigy limited, animal production, 4332 stein, switzerland
-ciba-geigy limited, basle, switzerland, toxicology ii. laboratories, animal facilities of toxicology ii. laboratories of residue analysis unit, agricultural division ciba-geigy limited, basle.
-ciba-geigy limited, metabolism and ecology department, r&d plant protection agricultural division, basle, switzerland
-ciba-geigy limited, plant protection division, ch-4002 basle, switzerland
-ciba-geigy limited, research and development department, product safety, safety evaluation, basle, switzerland.
-ciba-geigy limited, tierfarm, 4334 sisseln, switzerland
-ciba-geigy ltd. ch-4002 basle, switzerland
-ciba-geigy ltd., basel, switzerland
-ciba-geigy ltd., basel, switzerland,
-ciba-geigy ltd., basle, ch
-ciba-geigy ltd., genetic toxicology, basel, switzerland
-ciba-geigy,greensboro, united states
-citoxlab france
-covance laboratories inc.9200 leesburg pike, vienna, virginia 22182
-covance laboratories limited, harrogate, uk
-covance laboratories ltd., north yorkshire, uk.
-covance laboratories, harrogate, united kingdom
-cultures maintained at wildlife international ltd. laboratories
-division of toxicology,  institute of environmental toxicology
-eba inc.
-eba inc., snow camp, usa
-eg&g bionomics
-epl inc., research triangle
-eurofins agroscience services chem sas, vergèze, france
-experimental toxicology, ciba-geigy limited, 4332 stein, switzerland
-fine organics limited, seal sands, middlesbrough ts2 1ub, uk
-genetic toxicology, novartis crop protection ag, ch-4002 basel, switzerland
-granja perrone, são bernardo do campo - sp – brazil
-harlan (ad zeist, the netherlands).
-harlan france, zi le malcourlet, 03800 gannat / france
-harlan laboratories b.v. kreuzelweg 53 5961 nm horst / the netherlands
-harlan laboratories b.v. postbus 6174 5960 ad horst / the netherlands
-harlan laboratories b.v., kreuzelweg 53, 5961 nm horst / the netherlands, postbus 6174, 5960 ad horst / the netherlands
-harlan laboratories ltd., itingen, switzerland, d24665
-harlan sprague dawley, inc., madison, wi.
-harlan uk, shaw’s farm, blackthorn, bicester, oxon, ox6 0tp
-harlan winkelmann gmbh, d-33178 borchen, germany
-hazleton wisconsin
-hazleton wisconsin, inc.
-hazleton wisconsin, inc., 3301 kinsman boulevard, madison, wisconsin
-houghton springs fish farm, dorset, uk
-huntingdon research centre ltd, cambridgeshire, england
-huntingdon research centre ltd., huntingdon, united kingdom
-huntingdon research centre ltd., p.o. box 2, huntingdon, cambridgeshire, pe18 6es, england
-ibc manufacturing co., memphis, tn, usa
-j. cole, the county game farms, ashford, kent, england
-jealott’s hill international, bracknell, berkshire, united kingdom
-jiangsu huifeng agrochemicals co. ltd.
-kleintierfarm madoerin ag, ch-4414 fuellinsdorf
-m & m quail farm, 4090 campbell road, gillsville, ga 30543, u.s.a.
-maryland exotic birds of pasadena, maryland usa
-max (rudong) chemical co ltd
-morse laboratories llc, 1525 fulton avenue, sacramento, ca 95825 usa
-mount lassen trout farms, california
-mr j. coles, the country game farms, ashford, kent, england.
-mt. lassen trout farm, rt. 5, box 36, red bluff, california 98080
-nichols rabbitry inc. ; lumberton, tx
-nichols rabbitry inc; lumberton, tx., us
-notox b.v., hertogenbosch, netherlands
-novartis crop protection ag, basel, switzerland ciba-geigy ltd., basel, switzerland
-novartis crop protection ag, product portfolio management, environmental safety, ecotoxicology, ch-4002 basel, switzerland
-organics limited, middlesbrough, united kingdom
-osage catfish./box 222/missouri 65065/usa
-osage catfisheries inc., lake road 54-56, route 4, box 1500, osage beach, mo65065, usa
-p. hohler / ch-4341 zeiningen, switzerland
-p. hohler, trout breeding station zeiningen, switzerland
-park, nc, usa
-plant protection division ciba-geigy limited basle, switzerland. genetic toxicology cibageigy limited basle, switzerland
-product safety laboratories, east brunswick, new jersey 08816-3206, usa
-product safety labs, east brunswick, usa
-rcc - biological research laboratories, füllinsdorf, switzerland,
-rcc cytotest cell research gmbh, rossdorf, germany
-rcc ltd, environmental chemistry & pharmanalytics, ch-4452 itingen / switzerland
-rcc ltd, itingen, switzerland
-rcc ltd, laboratory animal services, wölferstrasse 4, 4414 füllinsdorf, switzerland
-rcc ltd., itingen, switzerland,
-rcc ltd., itingen, switzerland, b18966, t009636-06
-rcc ltd., laboratory animal services, ch-4414 füllinsdorf, switzerland
-rcc ltd., toxicology, wölferstrasse 4, ch-4414 füllinsdorf, switzerland
-rcc ltd., zelgliweg 1, 4452 itingen, switzerland
-rcc, cytotest cell research gmbh (rcc-ccr), in den leppsteinwiesen19, 64380 rossdorf, germany
-research department, pharmaceuticals division, ciba-geigy corporation, 556 morris avenue, summit, new jersey 07901
-ricerca, inc., ohio, usa
-rodent breeding unit, alderley park, macclesfield, uk
-sequani limited, bromyard road, ledbury, herefordshire, hr8 1lh, united kingdom
-sequani limited, ledbury, united kingdom
-sequani limited, ledbury, united kingdom,
-sipcamadvan, durham, nc, usa
-smithers viscient, 790 main street, wareham, ma 02571-1037 usa
-smithers viscient, 790 main street, wareham, ma, usa
-smithers viscient, 790 main street, wareham, massachusetts 02571 usa
-smithers viscient, 790 main street, wareham, massachusetts 02571-1037, usa
-source tierfarm sisseln, switzerland
-southwest bio-labs, inc.401 n. 17th street, suite 11, las cruces, nm 88005 usa.
-spring creek trout hatchery, lewistown, montana, usa
-springborn (europe) ag, horn, switzerland
-springborn laboratories inc., wareham, usa
-springborn laboratories, inc. 790 main street wareham, massachusetts 02571
-springborn laboratories, inc. environmental sciences division, 790 main street, wareham, 02571, usa massachusetts
-springborn laboratories, inc.,
-springborn laboratories, inc., health and environmental sciences, 790 main street, wareham, massachusetts, 02571-1075, usa
-springborn life sciences inc.,
-springborn smithers laboratories, wareham, usa
-stillmeadow inc. study number 9062-05,
-stillmeadow inc., sugar land, united states,
-stillmeadow inc., sugarland tx, usa
-stillmeadow inc., sugarland tx, usa, 8065-04 8321-03
-stillmeadow, inc, 12852 park one drive, sugar land, tx 77478, us
-stillmeadow, inc., 12852 park one drive, sugar land, tx 77478, usa
-syngenta - jealott’s hill, bracknell, united kingdom
-syngenta -jealott’s hill international research centre, uk
-syngenta central toxicology laboratory, alderley park, macclesfield, cheshire, uk
-syngenta crop protection, llc, greensboro, nc, usa
-syngenta crop protection, llc, greensboro, usa
-syngenta crop protection, monthey, switzerland
-syngenta ctl, alderley park, macclesfield, cheshire, sk10 4tj, uk
-syngenta – jealott’s hill international, bracknell, berkshire, united kingdom
-syngenta, jealott’s hill, international research station, bracknell, rg42 6ey, united kingdom
-texas animal specialties, humble, tx
-texas animal specialties, humble, tx, us
-toxigeneticsinc. decatur, il, us
-uk. charles river
-veterinary health research pty ltd, nsw, australia
-vischim srl, c/o lewis & harrison, llc, washington, dc, usa
-vischim srl, milano, italy
-wil research laboratories, llc, 1407 george road.ashland, oh, usa
-wil research laboratories, llc, ashland, oh, usa
-wil research laboratories, llc, ashland, oh, usa,
-wil research, 1407 george road, ashland, oh, 44805-8946, usa
-wil research, llc, 1407 george road, ashland, oh 44805-8946, usa
-wildlife international a division of eag inc. 8598 commerce drive easton, md 21601
-wildlife international ltd. cultures, 8651 brooks drive, easton, maryland 21601
-wildlife international ltd., 8598 commerce drive, easton, maryland 21601, usa
-wildlife international ltd., 8598 commerce drive, maryland 21601, usa
-wildlife international ltd., easton md, usa
-wildlife international ltd., easton, maryland 21601, usa
-wildlife international ltd., easton, usa
-wildlife international ltd., maryland, us
-wildlife international ltd., maryland, usa
-wildlife international, 8598 commerce drive, easton, md 21601 usa
-wildlife international, a division of eag inc., 8598 commerce drive, easton, md 21601 usa
-wise d.r. & wise r.e., monkfield, bourn, cambridgeshire, england
-zeneca agrochemicals, jealott’s hill, united kingdom
-zentralinstitut fur versuchstierzucht gmbh, hannover, germany",
-Syngenta Ltd., Jealott’s Hill International Research Centre, Bracknell, Berkshire, RG42 6EY, UK.
-Sequani Limited, Bromyard Road, Ledbury, Herefordshire, HR8 1LH, UK.
-Harlan Cytotest Cell Research GmbH (Harlan CCR), In den Leppsteinswiesen 19, 64380 Rossdorf, Germany
-Harlan Laboratories Ltd, Itingen, Switzerland.
-Bioassay Labor fuer biologische Analytik GmbH INF 515, 69120 Heidelberg, Germany
-Syngenta Crop Protection Ltd.
-Syngenta, Jealott’s Hill, Bracknell, United Kingdom
-Charles River Laboratories, Preclinical Services, Tranent (PCS-EDI) Edinburgh, EH33 2NE, UK
-CXR Biosciences, 2, James Lindsay Place, Dundee Technopole, Dundee, DD1 5JJ, Scotland, UK
-CiToxLAB Hungary Ltd. H-8200 Veszprém, Szabadságpuszta Hungary
-Charles River, Tranent, Edinburgh, EH33 2NE, UK
-Charles River Laboratories Edinburgh Ltd., Tranent, Edinburgh, EH33 2NE, UK
-BASF SE; Ludwigshafen/Rhein; Germany Fed.Rep.
-Leatherhead Food Research (LFR), Molecular Sciences Department, Randalls Road, Leatherhead, Surrey, KT22 7RY, UK
-Syngenta, Jealott’s Hill, Bracknell, United Kingdom
-Department of Veterinary & Biomedical Sciences, 101 Life Sciences Building, Penn State University, University Park, PA 16802, USA
-CiToxLAB Hungary Ltd., H-8200 Veszprém, Szabadságpuszta, Hungary
-SafePharm Laboratories Ltd, Shardlow Business Park, Shardlow, Derbyshire, UK
-Harlan Laboratories Ltd., Zelgliweg 1, 4452 Itingen, Switzerland
-RCC, Cytotest Cell Research GmbH (RCC-CCR), In den Leppsteinswiesen 19, 64380 Rossdorf, Germany
-Harlan, Cytotest Cell Research GmbH (Harlan CCR), In den Leppsteinswiesen 19, 64380 Rossdorf, Germany
-Harlan Laboratories Ltd. Zelgliweg 1, CH-4452 Itingen / Switzerland
-Quotient Bioresearch (Rushden) Ltd., Pegasus Way, Crown Business Park, Rushden, Northamptonshire, NN10 6ER, UK
-Charles River Laboratories Edinburgh, Ltd., Elphinstone Research Centre, Tranent, East Lothian, EH33 2NE, United Kingdom
-CiToxLAB Hungary Ltd. H-8200 Veszprém, Szabadságpuszta, Hungary
-Harlan Cytotest Cell Research GmbH, In den Leppsteinswiesen 19, 64380 Rossdorf Germany
-Charles River, Tranent, Edinburgh, EH32 2NE, UK
-Charles River Laboratories Edinburgh Ltd, Tranent, Edinburgh, EH33 2NE, UK
-Harlan Cytotest Cell Research GmbH, (Harlan CCR), In den Leppsteinswiesen 19, 64380 Rossdorf, Germany
-Charles River UK Limited, Margate, Kent, UK
-RCC Ltd., Biotechnology & Animal Breeding Division, 4414 Fuellinsdorf, Switzerland
-Charles River (UK) Ltd., Margate, Kent, CT9 4LT, England
-Charles River Ltd., Margate, Kent, United Kingdom
-Charles River UK Ltd, Manston Road, Margate, Kent CT9 4LT, England, UK
-Syngenta Crop Protection, Toxicology, 4332 Stein, Switzerland
-Safepharm Laboratories Limited, Shardlow Business Park, Shardlow, Derbyshire, DE72 2GD, United Kingdom
-Sequani Ltd, Bromyard Road, Ledbury, Herefordshire, HR8 1LH, United Kingdom
-Central Toxicology Laboratory, Alderley Park, Macclesfield, Cheshire, SK10 4TJ, UK
-Charles River UK
-Department of Veterinary & Biomedical Sciences, Penn State University
-Syngenta Ltd. Jealott’s Hill International Research, Bracknell, Berks RG42 6EY
-Charles River Laboratories, Research Models and Services Germany GmbH; Sandhofer Weg 7, 97633 Sulzfeld, Germany
-Novartis Crop Protection AG, Toxicology, 4332 Stein, Switzerland
-BRL Biological Research Laboratories Ltd., Wölferstrasse 4, 4414 Füllinsdorf, Switzerland
-B&K Universal Ltd, Grimston, Aldbrough, Hull, HU11 4QE, East Yorkshire, UK
-B&K Universal Ltd, Grimston, Aldborough, Hull, UK
-Nunc GmbH & Co. KG, 65203 Wiesbaden, Germany
-Fluka, 89203 Neu-Ulm, Germany
-MERCK, 64293 Darmstadt, Germany
-Charles River Laboratories, Research Models and Services Germany GmbH; Sandhofer Weg 7, 97633 Sulzfeld, Germany
-Animal Production, Novartis Pharma AG, 4332 Stein, Switzerland
-RCC Ltd., Biotechnology & Animal Breeding Division, 4414 Fuellinsdorf, Switzerland.
-SYSTAT Software, Inc., 501, Canal Boulevard, Suite C, Richmond, CA 94804, USA
-Safepharm Laboratories Limited, Shardlow Business Park, Shardlow, Derbyshire, DE72 2GD, United Kingdom
-Charles River (UK) Limited, Margate, Kent, CT9 4LT, England
-CXR Biosciences, 2 James Lindsay Place, Dundee Technopole, Dundee, DD1 5JJ, Scotland, UK
-Granja Perrone, São Bernardo do Campo - SP – Brazil
-Harlan Sprague-Dawley, Inc. Houston/Texas
-P. Hohler, trout breeding station Zeiningen, 4314 Zeiningen, Switzerland
-Spring Creek trout hatchery, Lewistown, Montana, USA
-Springborn laboratories culture facility
-Springborn culture
-University of Texas
-Institute for Plant Physiology, University of Göttingen, 37073 Göttingen, Germany
-Bayer CropScience AG, 40789 Monheim, Germany
-Koppert B. V. Berkel en Rodenrijs, Nederland
-Bio-Test Labor GmbH, Sagerheide, Germany
-Ciba-Geigy
-Ciba-Geigy Ltd.
-Harlan Laboratories Ltd., Itingen, Switzerland, D24643
-Springborn Laboratories Inc., Wareham, USA
-Springborn Laboratories (Europe) AG
-Syngenta Eurofins - GAB, Niefern Öschelbronn, Germany
-Syngenta Eurofins Agroscience Services EcoChem GmbH, N-Osch., Germany
-Novartis Crop Protection AG, Basel, CH
-Springborn (Europe) AG, Horn, Switzerland
-Springborn Smithers Laboratories (Europe) AG, Horn, Switzerland
-Syngenta Crop Protection AG, Basel, Switzerland
-GAB Biotechnologie GmbH, Niefern, Germany
-BioChem Agrar, Gerichshain, Germany
-AgroChemex Ltd, Manningtree, United Kingdom
-Ciba-Geigy Ltd., Basel, Switzerland
-Ciba-Geigy Muenchwilen AG, Muenchwilen, Switzerland
-Novartis Crop Protection Münchwilen AG, Münchwilen, Switzerland
-Novartis Crop Protection AG, Basel, Switzerland
-Ciba-Geigy Muenchwilen AG, Muenchwilen, Switzerland
-Charles River Laboratories, Research Models and Services Germany GmbH; Sandhofer Weg 7, 97633 Sulzfeld, Germany
-Alderley Park
-Alderley Park Swiss
-Stillmeadow, Inc., 12852 Park One Drive, Sugar Land, TX 77478, USA
-Texas Animal Specialties, Humble, TX
-Nichols Rabbitry Inc. ; Lumberton, TX
-Charles River Laboratories., Wilmington, MA
-Charles River Laboratories Edinburgh Ltd., Elphinstone Research Centre, Tranent, East Lothian, EH33 2NE
-Syngenta Crop Protection, Monthey, Switzerland
-Syngenta Crop Protection, Münchwilen, Switzerland
-Fine Organics Limited, Middlesbrough, United Kingdom
-Fine Organics Limited, Seal Sands, Middlesbrough TS2 1UB, UK
-Syngenta Crop Protection, Inc., Greensboro, USA
-Syngenta Technology & Projects, Huddersfield, United Kingdom
-Syngenta Biosciences Pvt. Ltd., Ilhas Goa, India
-Syngenta - Process Hazards Section, Huddersfield, United Kingdom
-Syngenta Walloon Agricultural Research Centre, Gembloux, Belgium , 21764
-Syngenta Crop Protection, Münchwilen, Switzerland, 300052719
-Syngenta Crop Protection Münchwilen AG, Münchwilen, Switzerland, 109747
-Syngenta Crop Protection, Münchwilen, Switzerland, 300073294
-Syngenta - Jealott’s Hill, Bracknell, United Kingdom RCC Ltd., Itingen, Switzerland, B18977, T003446-06
-Syngenta - Jealott’s Hill, Bracknell, United Kingdom RCC Ltd., Itingen, Switzerland, B18966, T009636-06
-RCC Cytotest Cell Research GmbH, Rossdorf, Germany, RCC 107662
-Syngenta Syngenta - Jealott’s Hill, Bracknell, United Kingdom,
-RCC Cytotest Cell Research GmbH, Rossdorf, Germany
-WIL Research Laboratories, LLC, Ashland, OH, USA
-Charles River Laboratories, Edinburgh, United Kingdom, 36955
-Syngenta Crop Protection AG, Basel, Switzerland Stillmeadow Inc., Sugarland TX, USA
-Novartis Crop Protection Inc., Greensboro, USA
-Syngenta - Jealott’s Hill, Bracknell, United Kingdom
-Eurofins - ADME Bioanalyses, Vergeze, France
-BioChem GmbH, Cunnersdorf, Germany
-Syngenta Syngenta Crop Protection, LLC, Greensboro, NC, USA
-Syngenta Eurofins Agroscience Services Chem SAS, Vergèze, France
-Syngenta Innovative Environmental Services, Witterswil, Switzerland
-Ricerca Biosciences, LLC, Concord, OH, USA
-Dr Knoell Consult GmbH, Mannheim, Germany
-RCC Umweltchemie GmbH & Co. KG, Rossdorf, Germany
-JSC International Ltd., Harrogate, United Kingdom
-Wildlife International Ltd., Easton, Maryland 21601, USA
-Syngenta Crop Protection, LLC, Greensboro, NC, USA
-Novartis - Greensboro, Greensboro, USA
-Smithers Viscient, 790 Main Street, Wareham, MA, USA
-Syngenta Cambridge Environmental Assessments, United Kingdom
-Ciba-Geigy Basel, Oekotoxikologie, Basel, Switzerland
-RCC Ltd., Itingen, Switzerland
-IBACON GmbH, Rossdorf, Germany
-Envigo Research Limited, Shardlow, UK
-Syngenta Crop Protection Münchwilen AG, Münchwilen, Switzerland
-Ciba-Geigy Münchwilen AG, Münchwilen, Switzerland
-Huntingdon Research Centre Ltd., Huntingdon, United Kingdom
-Syngenta Technology & Projects, Huddersfield, United Kingdom
-Harlan Laboratories Ltd., Shardlow, Derbyshire, UK
-Dr. Specht & Partner Chem. Laboratorien GmbH, Hamburg, Germany
-Institut Fresenius, Taunusstein, Germany
-Syngenta - Jealott’s Hill International, Bracknell, Berkshire, United Kingdom
-Ciba-Geigy Corp., Greensboro, USA
-CIP Chemisches Institut Pforzheim GmbH, Pforzheim, Germany
-Charles River Laboratories Edinburgh Ltd, Tranent, EH33 2NE, UK
-Hazleton Laboratories, Madison, USA
-Eurofins BioPharma, Planegg, Germany, 150556
-Syngenta Environ. Health Center, Farmington, USA
-Centre International de Toxicologie C.I.T., Evreux, France
-Toxalim, Research Centre in Food Toxicology, F- 31027 Toulouse, France
-Harlan Laboratories Ltd., Shardlow, Derbyshire, UK
-CRS GmbH GmbH, In den Leppsteinswies en 19, 64380 Rossdorf Germany
-Environ. Health Center, Farmington, USA
-Ciba-Geigy Corp., Summit, USA
-Ciba-Geigy Basel, Genetische Toxikologie, Basel, Switzerland
-Ciba-Geigy Ltd., Stein, Switzerland
-Novartis Crop Protection AG, Stein, Switzerland
-Central Toxicology Laboratory (CTL), Cheshire, United Kingdom
-Sequani Limited, Bromyard Road, Ledbury, Herefordshire, HR8 1LH, United Kingdom
-Brixham Environmental Laboratory, Brixham, United Kingdom
-Springborn Smithers Laboratories, Horn, Switzerland
-Huntingdon Research Centre, Cambridgeshire, United Kingdom
-Mambo-Tox Ltd., Southampton, United Kingdom
-MITOX Consultants, Amsterdam, Netherlands
-Charles River Aquaria, Margate, UK
-Brixham Environmental Laboratory, Brixham, UK
-O.Keller, Mörschwil, CH
-Huntingdon Life Sciences Ltd., Huntingdon, UK
-BTL Bio-Test Labor GmbH, Sagerheide, Germany
-Mambo-Tox Ltd., Southampton, UK
-Mambo-Tox Ltd. 2 Venture Road, University Science Park, Southampton SO16 7NP, United Kingdom
-BioChem GmbH, Germany
-PK Nützlingszuchten, Welzheim, Germany
-BioChem agrar, Germany
-Sautter & Stepper, Ammerbuch, Germany
-Koppert, The Netherlands
-Kraut & Rubeen (Doris Haber), Zeilstraße 40, 64367 Mühltal-Frankenhausen, Germany
-Springborn Laboratories (Europe) AG, Seestrasse 21, CH-9326 Horn, Switzerland
-Biologische Bundesanstalt (BBA), Braunschweig, Germany
-Institut für Biologische Analytik und Consulting, IBACON GmbH, Arheilger Weg 17, 64380 Rossdorf, Germany
-Abandoned vineyard, Northern Italy
-Syngenta Limited, Cheshire, United Kingdom
-Agrochemex, Lawford, United Kingdom
-Staphyt, Inchy en Artois, France
-Dermal Technology Laboratory Ltd., Staffordshire, UK
-Ciba Agriculture, Whittlesford, United Kingdom
-Bayer Crop Science AG, Monheim, Germany
-tier3 solutions GmbH, Leichlingen, Germany
-Mambo-Tox. Ltd., Southampton, United Kingdom
-Syngenta Crop Protection AG, Stein, Switzerland
-Stillmeadow Inc, Sugar Land, TX 77478, US
-Texas Animal Specialities, Humble, TX, US
-CiToxLAB, 8200 Veszprem, Szabadsagpuszta, Hungary
-Syngenta Ltd, Jealott's Hill International Research Centre, Bracknell, Berkshire, RG42 6EY, United Kingdom
-Stillmeadow, Inc, 12852 Park One Drive, Sugar Land
-Syngenta Central Toxicology Laboratory, Alderley Park, Macclesfield, Cheshire, UK
-Syngenta Limited, Alderley Park, Macclesfield, Cheshire, SK10 4TJ
-Nichols Rabbitry Inc; Lumberton, TX., US
-AgroChemex International Ltd, Aldhams Farm Research Station, Lawford, Essex, UK
-Ciba Agriculture, Whittlesford, Cambridge, UK
-Ricerca Inc., Department of Residue Analysis, Painesville OH, USA
-Staphyt, 23 rue de Moeuvres, F-62860 Inchy en Artois, France
-Dermal Technology Laboratory Ltd., Med IC4, Keele University Science and Business Park, Keele, Staffordshire, ST5 5NL, United Kingdom
-Tier3 solutions GmbH, Kolberger Strasse 61-63 51381 Leverkusen, Germany
-RCC Ltd, Environmental Chemistry & Pharmanalytics, CH-4452 Itingen / Switzerland
-GAB Biotechnologie GmbH & IFU Umweltanalytik GmbH, Niefern-Öschelbronn, Germany
-Biochem agrar, Germany
-Bienenfarm Kern GmbH, Am Rehbacher Anger 10, 04249 Leipzig, Germany
-Joaquin Cordero, Paseo de Colón No. 19, 41370 Cazalla (Sevilla), Spain
-Mambo-Tox Ltd, Southampton, UK
-GAB Biotechnologie GmbH & IFU Umweltanalytik GmbH, Niefern-Öschelbronn, Germany
-Innovative Environmental Services (IES), Benkenstrasse 260, 4108 Witterswil, Switzerland
-BioChem agrar GmbH, Kupferstraße 6, 04827 Gerichshain, Germany
-RCC - Biological Research Laboratories, Füllinsdorf, Switzerland, 859442
-RCC Ltd., Toxicology, Wölferstrasse 4, CH-4414 Füllinsdorf, Switzerland
-RCC Ltd., Laboratory Animal Services, CH-4414 Füllinsdorf, Switzerland
-Charles River Laboratories France, BP 0109, F-69592 L’Arbresle
-Charles River Deutschland GmbH, Stolzenseeweg 32-36, D-88353 Kisslegg / Germany
-Syngenta CTL, Alderley Park, Macclesfield, Cheshire, SK10 4TJ, UK
-Harlan UK, Shaw’s Farm, Blackthorn, Bicester, Oxon, OX6 0TP
-Syngenta Central Toxicology Laboratory, UK
-RCC Ltd., Toxicology, Wölferstrasse 4, CH- 4414 Füllinsdorf, Switzerland
-RCC Ltd, Itingen, Switzerland
-P. Hohler, trout breeding station Zeiningen, Switzerland
-SAG, Institute for Plant Physiology, University of Göttingen, Germany
-GAB Biotechnologie GmbH, Niefern-Öschelbronn, Germany
-Beekeeper Mr. Berthold Nengel, Brückenstraße 12, 56348 Dahlheim, Germany
-Syngenta Crop Protection, Münchwilen, Switzerland, CHMU140561
-Syngenta Crop Protection, Münchwilen, Switzerland
-Sequani Limited, Ledbury, United Kingdom, BFI0516
-PTRL Europe, Ulm, Germany
-SGS Institut Fresenius GmbH, Taunusstein, Germany
-CEM Analytical Services Ltd (CEMAS) - Berkshire, UK
-PTRL Europe, Ulm, Germany
-Sequani Limited, Ledbury, United Kingdom
-SGS Institut Fresenius GmbH, Taunusstein, Germany
-CEM Analytical Services, UK
-Eurofins Agroscience Services Chem SAS, Vergà ̈ze, France
-Novartis Services AG, Basel, Switzerland
-BSL Bioservice Scientific, Planegg, Germany
-Envigo CRS GmbH, Rossdorf, Germany
-Envigo CRS GmbH, In den Leppsteinswiesen 19, 64380 Rossdorf, Germany
-BASF Ltd., Ludwigshafen, Germany
-ALS Laboratory Group, Edmonton, Alberta, Canada
-Syngenta Crop Protection, Inc., Greensboro, USA
-ADME - Bioanalyses, Vergeze, France
-Battelle UK Ltd., Ongar, United Kingdom
-SGS Institut Fresenius GmbH
-Novartis Agro GmbH, Frankfurt, Germany
-Supervision & Test Center Pesticide Safety Evaluation, China
-T. R. Wilbury Laboratories, Inc., Marblehead, MA, USA
-CEMAS, North Ascot, United Kingdom
-EAG Laboratories PTRL Europe GmbH, Germany
-Syngenta Crop Protection Inc., USA
-Syngenta Crop Protection Inc., 410 Swing Road, Greensboro, NC 27409, USA
-Huntingdon Research Centre Ltd., UK
-Huntingdon Research Centre Ltd., England
-T.R. Wilbury Laboratories, Inc., USA
-Wildlife International Ltd., USA
-RCC Ltd, Switzerland
-RCC Ltd. Environmental Chemistry & Pharmanalytics Division CH-4452 Itingen/Switzerland
-Harlan Laboratories Ltd., Switzerland
-CIBA-GEIGY Ltd., Switzerland
-Syngenta Crop Protection AG, Basel , Switzerland
-Syngenta Crop Protection LLC, Greensboro, USA
-PTRL Europe GmbH, Helmholtzstr. 22, Science Park, Ulm, Germany
-PTRL Europe GmbH, Germany
-CEM Analytical Services Ltd (CEMAS), Imperial House, Oaklands Business Centre, Oaklands Park, Wokingham, Berkshire, RG41 2FD UK
-SGS INSTITUT FRESENIUS GmbH
-Syngenta Ltd, Jealott’s Hill International Research Centre, Bracknell, Berkshire, RG42 6EY, UK
-Fraunhofer Institute for Molecular Biology and Applied Ecology, IME, Auf dem Aberg 1, 57392 Schmallenberg, Germany
-Eurofins Agroscience Services Chem SAS, 75B, Avenue du Pascalet, 30310 Vergèze, France
-Innovative Environmental Services (IES) Ltd, Benkenstrasse 260, 4108 Witterswil, Switzerland
-BSL Bioservice, Scientific Laboratories GmbH, Behringstrasse 6/8, 82152 Planegg, Germany
-RCC Ltd, Zelgliweg 1, CH-4452 Itingen, Switzerland
-RCC Ltd, Laboratory Animal Services, CH-4414 Fuellinsdorf
-Harlan Cytotest Cell Research GmbH, In den Leppsteinswiesen 19, 64380 Rossdorf, Germany
-Ciba-Geigy Limited, Basel, Switzerland
-BASF SE, Experimental Toxicology and Ecology, 67056 Ludwigshafen, Germany
-Ciba-Geigy Limited, Animal Production, 4332 Stein, Switzerland
-RCC Ltd. Biotechnology & Animal Breeding Division, 4414 Füllinsdorf, Switzerland
-Syngenta Ltd. Jealott’s Hill International Research Centre, Bracknell, Berks RG42 6EY
-WIL Research Laboratories, LLC, 1407 George Road, Ashland, Ohio 44805-8946, USA
-Charles River Laboratories Inc., Kingston, New York, USA
-Syngenta, Jealott’s Hill International Research Centre, Bracknell, United Kingdom
-Battelle UK Ltd
-D.R. & R.E. Wise, Monkfield, Bourn, Cambridgeshire, England
-Wildlife International. 8598 Commerce Drive, Easton, MD 21601 USA
-Maryland Exotic Birds of Pasadena, MD 21122
-Mr D. R. Wise, Monkfield, Bourn, Cambridgeshire, England
-Cambridge Environmental Assessments, Battlegate Road, Boxworth, Cambridgeshire, CB23 4NN, UK
-J. Coles, The County Game Farms, Ashford, Kent, England
-Osage Catfisheries, MO 65 065, USA
-Supervision and Test Center for Pesticide Safety Evaluation and Quality Control, 600 Shenliao Road, Tiexi District, Shengyang 110141, Liaoning Province, P.R. China
-Syngenta, Jealott’s Hill International Research Centre, Bracknell, Berkshire, RG42 6EY
-Harlan Laboratories Ltd., 4452 Itingen, Switzerland
-Ciba-Geigy Ltd., Product Safety, Ecotoxicology, CH-4002 Basel, Switzerland
-P. Hohler, CH-4314 Zeiningen
-Cambridge Environmental Assessments, Battlegate Road, Boxworth, Cambridgeshire, CB23 4NN/UK
-Wildlife International, A Division of EAG Inc. 8598 Commerce Drive Easton, MD 21601 USA
-Novartis Crop Protection AG, Kanton Aargau, Switzerland.
-RCC Ltd, CH-4452 Itingen, Switzerland
-CEMAS, North Ascot, Berkshire, UK
-Wilbury Laboratories Inc, 40 Doaks Lane, Marblehead, Massachusetts
-P. Cummins Oyster Company, Pasadena, Maryland
-Harlan Laboratories Ltd, Zelgliweg 1, 4452 Itingen/Switzerland
-PK Nützlingszuchten, D-73642 Welzheim, Germany
-Institut für Biologische Analytik und Consulting IBACON GmbH Arheilger Weg 17, 64380 Rossdorf, Germany
-ABC Laboratories Inc., Analytical Chemistry and Field Services, 7200 E. ABC Lane, Columbia, Missouri
-Ciba-Geigy Corporation, Farmington, CT, USA
-Syngenta Ltd. Jealott’s Hill, Bracknell, United Kingdom
-Eurofins Agroscience Services EcoChem GmbH, N- Osch., Germany
-Ciba-Geigy Limited, Animal Production Unit, Basle, Switzerland
-Ciba-Geigy Limited, Basle, Switzerland
-Charles River Laboratories, Raleigh, NC, USA
-Charles River (UK) Limited
-Harlan Sprague Dawley, Inc., Madison, WI
-CIBA-GEIGY Limited, Animal Production, 4332 Stein, Switzerland
-CIBA-GEIGY Limited, 4332 Stein, Switzerland
-Kleintierfarm Madoerin AG, CH-4414 Fuellinsdorf
-CIBA-GEIGY Limited, Tierfarm, 4334 Sisseln, Switzerland
-Animal production, CIBA-GEIGY Limited, 4332 Stain/Switzerland
-Environmental Health Centre (EHC), 400 Farmington Avenue, Farmington, CT 06032
-Charles River Laboratories, Kingston, NY
-Harlan (Ad Zeist, the Netherlands)
-Animal Production CIBA-GEIGY Limited 4332 Stein / Switzerland
-Tierfarm, Sisseln, Switzerland
-Zen-tralinstitut fur Versuchstier-zucht GmbH, Hannover, Germany
-Charles River Laboratories, Portage, MI
-CIBA-GEIGY Limited, Basel, Switzerland
-Novartis Crop Protection AG, CH-4002 Basel, Switzerland
-RCC Ltd., Biotechnology and animal breeding division, Fullinsdorf, Switzerland
-Tierfarm Sisseln, Switzerland
-Charles River Breeding Laboratories, Raleigh, NC, USA
-Ciba-Geigy Corporation, Plant Protection Division, Environmental Health Center, 400 Farmington Avenue, Farmington, Connecticut 06032, USA
-Charles River Breeding Laboratories, Inc., Raleigh, North Carolina USA
-Charles River, 76410, Saint-Aubin-les-Elbeuf, France
-Charles River Laboratories, Inc., Raleigh, NC, USA
-WIL Research Laboratories, LLC, 1407 George Road, Ashland, OH 44805-8946 USA
-RCC Ltd., Biotechnology & Animal Breeding Division, 4414 Fȕllinsdorf, Switzerland
-Alderley Park, Macclesfield, Cheshire UK
-Rodent Breeding Unit, Alderley Park, Macclesfield, UK
-Harlan Winkelmann GmbH, D-33178 Borchen, Germany
-WIL Research Laboratories, LLC, 1407 George Road.Ashland, OH 44805-8946 USA
-Centre d’Elevage Charles River
-CIBA-GEIGY Limited, Experimental Toxicology, 4332 Stein/Switzerland
-Centre International de Toxicologie (C.I.T.), Miserey, 27005 Evreux, France
-Centre Internationale de Toxicologie, Miserey, 27005 Evreux, France
-CIBA-GEIGY Limited, Basle, Switzerland
-Harlan Laboratories Ltd, Shardlow Business Park, Shardlow, Derbyshire, DE72 2GD, UK
-Envigo CRS GmbH GmbH, In den Leppsteinswiesen 19, 64380 Rossdorf Germany
-Ciba-Geigy Ltd., Genetic Toxicology, Basel, Switzerland
-Toxalim, Research Centre in Food Toxicology, F-31027 Toulouse, France
-Ciba-Geigy Corp, Plant Protection Division, Environmental Health Center, 400 Farmington Avenue, Farmington, Connecticut 06032, USA
-Charles River France
-Charles River US
-WIL Research, LLC, 1407 George Road, Ashland, OH 44805-8946, USA
-Novartis Crop Protection AG, Toxicology, 4332 Stein Switzerland
-Syngenta Crop Protection, Health Assessment 2 Stein, 4332 Stein, Switzerland
-RCC Ltd. Biotechnology and Animal Breeding Division, 4414 Füllinsdorf, Switzerland
-Genetic Toxicology, Novartis Crop Protection AG, CH-40002 Basel, Switzerland
-RCC - Cytotest Cell Research GmbH In den Leppsteinswiesen 19, D- 64380 Roβdorf, Germany
-RCC - Cytotest Cell Research GmbH, In den Leppsteinswiesen 19, D-64380 Rofldorf, Germany
-Ciba-Geigy Limited, Animal production, 4332 Stein, Switzerland
-RCC Ltd., Zelgliweg 1, 4452 Itingen, Switzerland
-RCC Ltd, Laboratory Animal Services, Wölferstrasse 4, 4414 Füllinsdorf, Switzerland
-RCC Ltd, Laboratory Animal Services, 4414 Füllinsdorf, Switzerland
-CIBA-GEIGY Limited, Basle, Switzerland
-RCC Cytotest Cell Research GmbH (RCC-CCR), In den Leppsteinswiesen 19, 64380 Rossdorf, Germany
-RCC Cytotest cell Research GmbH, In den Leppsteinwiesen 19, Rossdorf, Germany
-Centre International de Toxicologie (CIT)
-C iba-Geigy
-Ciba-Geigy, Greensboro, North Carolina
-Ciba-Geigy Corp., Greensboro, United States
-Ciba-Geigy Vero Beach Research Center, Florida, USA
-Ciba-Geigy Corporation, Environ. Health Center, Farmington, United States
-Ciba-Geigy GmbH, Frankfurt a.Main, Germany
-Ciba-Geigy Corp., Greensboro, United States
-Wise D.R. & Wise R.E., Monkfield, Bourn, Cambridgeshire, England
-Mr J. Coles, The Country Game Farms, Ashford, Kent, England
-Maryland Exotic Birds of Pasadena, Maryland USA
-J. Cole, The County Game Farms, Ashford, Kent, England
-BC Potter, Rosedean, Woodhurst, Cambridgeshire, England
-M & M Quail Farm, 4090 Campbell Road, Gillsville, GA 30543, U.S.A
-Wildlife International A Division of EAG Inc. 8598 Commerce Drive Easton, MD 21601 USA
-M & M Quail Farm, 4090 Campbell Road, Gillsville, GA 30543 U.S.A
-China Agricultural University, No.2, Yuan Ming Yuan West Road, Haidian District, Beijing, 100193, P.R. China
-Mt. Lassen Trout Farm, Rt. 5, Box 36, Red Bluff, California 98080
-Bybrook Bass Hatchery, Connecticut
-CIBA-GEIGY Ltd. CH-4002 Basle, Switzerland
-Wildlife International Ltd. Cultures, 8651 Brooks Drive, Easton, Maryland 21601
-Aquatic bioassay laboratory, Baton Rouge, Louisiana
-P. Hohler/ CH-4314 Zeiningen, Switzerland
-Houghton Springs Fish Farm, Dorset, UK
-Cultures maintained at Wildlife International Ltd. Laboratories
-Aquatic Bio Systems, Inc., Fort Collins, Colorado
-Smithers Viscient, 790 Main Street, Wareham, Massachusetts 02571- 1037 USA
-Smithers Viscient, 790 Main Street, Wareham, Massachusetts 02571 USA
-Springborn laboratories
-Syngenta Ltd. Jealott’s Hill International Research Centre Bracknell, Berkshire, RG42 6EY United Kingdom
-Wildlife International Ltd., Maryland, USA
-Wildlife International Ltd., Easton, USA
-Smithers Viscient, 790 Main Street, Wareham, Massachusetts 02571 USA
-Brixham Environmental Laboratory, AstraZeneca UK Limited, Brixham, UK
-Springborn Laboratories Inc., Massachusetts 02571, USA
-Smithers Viscient, 790 Main Street, Wareham, MA 02571-1037, USA
-Wildlife International Ltd, Easton, MD, USA
-Wildlife International A Division of EAG Inc. 8598 Commerce Drive Easton, MD 21601 USA
-Smithers Viscient, 790 Main Street, Wareham, MA 02571-1037 USA
-Ciba-Geigy Corporation, Post Office Box 18300, Greensboro, NC 27419, USA
-Chesapeake Cultures, Hayes, Virginia
-Smithers Viscient, 790 Main Street, Wareham, Massachusetts 02571-1037 USA
-University of Sheffield, UK
-Blue Frog Scientific Limited, Scott House, South St. Andrew Street, Edinburgh, EH2 2AZ, UK
-MBL Aquaculture, Sarasota, Florida
-Bayer AG (Pflanzenschutz Umweltforschung, Institut für Oekobiologie, D- 5090 Leverkusen)
-Pflanzenphysiologisches Institut University, Nikolausberger Weg 180, D-3400 Göttingen, Germany
-Envigo Research Limited Shardlow Business Park, Shardlow, Derbyshire, DE72 2GD, UK
-Smithers Viscient, 790 Main Street, Wareham, Massachusetts 02571- 1037 USA
-Wildlife International Ltd., Easton, Maryland, USA
-David Francis, W.J. Mead Apiarist Supplies, Fowlmere, Cambridgshire
-RCC AG, Itingen, Switzerland
-Blades Biological Ltd, United Kingdom
-ECT Oekotoxikologie GmbH, Germany
-BioChem agrar, Labor für biologische und chemische, Analytik GmbH, Kupferstraße 6, 04827 Gerichshain, Germany
-RCC Umweltchemie AG, P.O. Box, CH-4452 Itingen/BL, Switzerland
-RCC Ltd, Environmental Chemistry & Pharmanalytics Division, CH-4452 Itingen, Switzerland
-BioChem agrar Labor für biologische und chemische, Analytik GmbH, Kupferstraße 6 04827 Gerichshain, Germany
-BioChem agrar, Labor für biologische und chemische, Analytik GmbH, Kupferstraße 6, 04827 Gerichshain, Germany
-Pan-Agricultural Labs, Inc. 32380 Avenue 10 Madera, CA 93638 USA
-Syngenta AG. Basel. Switzerland
-Deutsche Sammlung von Mikroorganismen und Zellkulturen GmbH, Inhoffenstraße 7 B, 38124 Braunschweig, Germany
-CIBA-GEIGY Ltd., Product Safety, Ecotoxicology, CH-4002 Basel, Switzerland
-Springborn Smithers Laboratories 790 Main Street Wareham, MA 02571-1037
-Syngenta crop protection AG, Research Biological science, Disease control, Stein
-Syngenta Biosciences Pvt. Ltd., Ilhas Goa, India
-Syngenta Technology & Projects, Huddersfield, United Kingdom
-Stillmeadow. Inc.. 12852 Park One Drive. Sugar Land. TX 77478. USA
-Texas Animal Specialties. Humble. TX
-Nichols Rabbitry Inc. ; Lumberton. TX
-Charles River Laboratories.. Wilmington. MA
-Charles River Laboratories Edinburgh Ltd.. Elphinstone Research Centre. Tranent. East Lothian. EH33 2NE
-Charles River Laboratories, Edinburgh, United Kingdom
-tier3 solutions GmbH
-tier3 solutions GmbH, Kolberger Str. 61-63, 51381 Leverkusen, Germany
-Bayer CropScience AG
-Syngenta, Jealott’s Hill International Research Centre, UK
-Brixham Environmental Laboratory, Brixham, Devon, TQ5 8BA, UK
-MITOX Consultants Science Park 408, 1098XH Amsterdam, The Netherlands
-Eurofins Agrosciences Services EcoChem GmbH, Eutinger Str. 24, 75233 Niefern-Öschelbronn, Germany
-Mambo-Tox Ltd., 2 Venture Road, Chilworth Science Park, Southampton SO16 7NP, United Kingdom
-Biochem agrar GmbH, Gerichshain, Germany
-“W. Neudorff GmbH KG”, An der Mühle 3, D- 31860 Emmertal
-BioChem Agrar, Kupferstraβe 6, 04827 Gerichshain, Germany
-Bayer CropScience AG, Monheim
-BioChem agrar, Labor für biologische und chemische Analytik GmbH, Kupferstraße 6, 04827 Gerichshain, Germany
-RIFCON GmbH, Hirschberg, Germany
-Dr K Thomae GMBH, Chemisch-pharmazeutische Fabrik, D-7950 Biberach, Riss
-Centre International de Toxicologie (C.I.T), Miserey, 27005 Evreux, France
-Centre d’Elevage Lebeau, 78950 Gambais, France
-CIBA-GEIGY Limited, Toxicology Services, Short-term Toxicology, 4332 Stein, Switzerland
-Ciba-Geigy Ltd., CH-4002, Basel, Switzerland
-Osage Catfish, Box 222, Missouri, USA
-Mambo-Tox Ltd., 2 Venture Road, University Science Park, Southampton, SO16 7NP
-Biologische Bundesanstalt (BBA), Berlin-Dahlem
-“Bayer CropScience AG” Monheim
-Zeneca Agrochemicals, Jealott’s Hill, United Kingdom
-Eurofins Agroscience Services Chem GmbH, Hamburg, Germany
-Harlan Cytotest Cell Research GmbH (Harlan CCR), Germany
-Smithers Viscient (ESG) Ltd, Harrogate, UK
-Covance Laboratories Limited, Harrogate, UK
-Central Toxicology Laboratory, Alderley Park, Macclesfield, Cheshire, UK
-Biological Services Section, Alderley Park, Macclesfield, Cheshire, UK
-Charles River
-Harlan Cytotest Cell Research GmBH, Rossdorf, Germany
-Syngenta Crop Protection, Inc., Greensboro, NC 27419, USA
-Cambridge Environmental Assessments, Battlegate Road, Boxworth, Cambridgeshire
-Central Toxicology Laboratory, Syngenta
-Harlan Laboratories Ltd. Zelgliweg,445 Itingen/Switzerland
-Tecsolve UK Ltd., Glendale Park, North Ascot, Berkshire
-Harlan Laboratories Ltd, Zelgliweg 1, 4452 Itingen, Switzerland
-Harlan Laboratories
-Katz Biotech AG, Baruth, Germany
-Mambo-Tox Ltd., 2 Venture Road, Chilworth Science Park, Southampton, SO16 7NP
-BioChem agrar, 04827 Gerichshain, Germany
-W. Neudorff, 31860 Emmerthal, Germany
-W. Neudorff GmbH KG, An der Mühle 3, 31860 Emmerthal, Germany
-BioChem agrar Labor für biologische und chemische Analytik GmbH, Kupferstraße 6 04827 Gerichshain, Germany
-“Biologische Bundesanstalt (BBA)”, Berlin-Dahlem
-BioChem agrar, Labor für biologische und chemische Analytik GmbH, Kupferstraβe 6, 04827 Gerichshain, Germany
-Syngenta Crop Protection, Münchwilen, Switzerland
-Ciba-Geigy Ltd., Basle, Switzerland
-Ciba-Geigy Corporation , Greensboro, NC, USA
-Ciba-Geigy Corp., Greensboro, NC, USA
-Nauchi, Shiraimachi, Inba-Gun, Chiba, Japan
-Animal Metabolism, Ciba-Geigy Ltd., Basle, Switzerland
-Hazleton Wisconsin, Inc. Madison, Wisconsin USA
-CiToxLAB Hungary Ltd, Szabadsagpuszta, Hungary
-Hazleton Wisconsin, Inc. Madison, Wis- consin USA
-Stillmeadow Inc., Sugar Land TX, USA
-Ciba-Geigy Corporation, Summit, NJ, USA
-Ciba-Geigy Corp., Environmental Health Center, Farmington, CT, USA
-Ciba-Geigy Limited, Pharmaceutical Division, 4002 Basel / Switzerland
-Ciba-Geigy Limited, Experimental Pathology, 4002 Basel/ Switzerland
-Ciba-Geigy Limited, Experimental Pathol- ogy, 4002 Basel / Switzerland
-Hazleton Wisconsin, Madison, WI, USA
-Ciba-Geigy Toxicology Services, ShortTerm Toxicology, 4332 Stein/ Switzerland
-Ciba-Geigy Limited, Experimental Pathology, 4002 Basel / Switzerland
-Hazleton Biotechnologies Company, Kensington, Maryland, USA
-Ciba-Geigy Limited, Genetic Toxicology, 4002 Basel / Switzerland
-Hazleton Washington, Inc., Vienna, Virginia 22182, USA
-Ciba-Geigy Limited, 4002 Basel / Switzerland
-Hazleton Raltech, Inc., a Subsidiary of Hazleton Laboratories America, Inc., Madison, Wisconsin, USA
-Experimental Pathology Laboratories, Research Triangle Park
-Toxicology/Cell Biology, Novartis Crop Protection Inc., Basel, Switzerland
-Toxigenics, Inc., Decatur, IL 62526, USA
-Argus Research Laboratories, Inc., Perkasie, PA, USA
-Argus Research Laboratories Inc., Horsham, Pennsylvania 19044, USA
-Ciba-Geigy Ltd.,Stein, Switzerland
-Ciba-Geigy Ltd., Genetic Toxicology, Basle, Switzerland
-Novartis Crop Protection AG, Stein, CH
-Safepharm Laboratories Ltd., Shadlow, United Kingdom
-Sandoz Agro Ltd., Department of Toxicology CH-4132 Muttenz, Switzerland
-Hazleton Washington, Inc. Vienna, Virginia, USA
-CXR Biosciences. Laboratory
-Ciba-Geigy Corp., Greensboro NC, USA
-Ciba-Geigy Ltd., Basel, CH
-Novartis Agro S.A., Aigues-Vives, F
-Ciba-Geigy SA, Rueil-Malmaison, F
-Novartis Agro S.A., Aigues-Vives, France
-Osage Catfisheries Inc., Osage Beach, Missouri 65065, USA
-Aquatic Biosystems Corvalis
-EPA, Corvalis, OR
-Ward’s Natural Science, ON
-Chilliwack Hatchery
-Sun Valley Trout Farm, Abbotsford BC
-Chilliwack Hatchery, BC
-P. Hohler, CH-4314 Zeiningen, Switzerland
-University of Sheffield , UK
-Wildlife International Ltd., Maryland, US
-Ciba-Geigy Ltd., Basle, CH
-Stillmeadow Inc., Sugar Land, United States
-Hazleton Wisconsin, Inc
-ToxigeneticsINc. Decatur, IL, US
-EG&G Bionomics
-Biospheric Inc., Rockville, USA
-Bionomics Aquatic Tox. Lab., Wareham, USA
-Springborn Laboratories Inc.
-Syngenta – Jealott’s Hill International, Bracknell, Berkshire, United Kingdom
-Wildlife International Ltd., Easton MD, USA
-Springborn Smithers Laboratories, Wareham, USA
-Springborn Life Sciences Inc
-Eg&G Bionomics (Fl), Pensacola, USA
-Harlan Laboratories Ltd., Itingen, Switzerland
-Solvias AG, Basel, Switzerland
-T.R. Wilbury Laboratories Inc., Massachusetts, USA
-Ciba-Geigy Ltd., Basle, CH
-Stillmeadow Inc., Sugar Land, TX, USA
-Syngenta Crop Protection, Munchwilen, Switzerland
-RCC - Biological Research Laboratories, Füllinsdorf, Switzerland
-Covance Laboratories, Harrogate, United Kingdom
-Battelle UK Ltd, Chelmsford, Essex, UK
-Zeneca Agrochemicals, Jealott’s Hill Research Station, Bracknell, Berkshire, UK
-Xenobiotic Laboratories, Inc., Plainsboro, USA
-Fraunhofer Institute, Schmallenberg, Germany
-PTRL West, Hercules CA, USA
-Eurofins Agroscience Services GmbH, Niefern-Öschel., Germany
-ICI Agrochemicals, Bracknell, Berkshire, United Kingdom
-Chemex International plc, Cambridge, United Kingdom
-BASF, Limburgerhof, Germany
-RIFCON, Leichlingen, Germany
-Eurofins - GAB, Niefern Öschelbronn, Germany
-River Thames, Maidenhead, Berkshire, UK
-Beach N o . 24, Hayling Island, Hampshire, UK
-Jealott’s Hill International Research Centre, Bracknell, Berkshire, RG42 6EY, UK
-Zeneca Agrochemical s, Jealott’s Hill, United Kingdom
-Zeneca Agrochemicals, Jealott’s Hill, United Kingdom
-Jealott’s Hill Research Station. Syngenta Crop protection AG
-Bayer CropScience, Monheim, Germany
-Huntingdon Life Sciences Ltd., Huntingdon, United Kingdom
-Eurofins Agroscience Services EcoChem GmbH, N- Osch., Germany
-Eurofins Agroscience Services EcoChem GmbH, NOsch., Germany
-Tier3 solutions GmbH, Germany
-Syngenta Crop Protection AG
-Jealott’s Hill Research Centre. Syngenta Crop protection AG
-RCC Umweltchemie GmbH & Co KG

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/rules.drl

@@ -1,66 +0,0 @@
-package drools
-
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section
-
-global Section section
-
-rule "0: Highlight Indicators"
-	when
-		eval(section.getEntities().isEmpty()==false);
-	then
-		section.highlightAll("VERTEBRATE");
-		section.highlightAll("NO_REDACTION_INDICATOR");
-	end
-
-
-rule "1: Redacted because Section contains Vertebrate"
-	when
-		eval(section.contains("VERTEBRATE")==true);
-	then
-		section.redact("NAME", 1, "Redacted because Section contains Vertebrate");
-		section.redact("ADDRESS", 1, "Redacted because Section contains Vertebrate");
-	end
-
-
-rule "2: Not Redacted because Section contains no Vertebrate"
-	when
-		eval(section.contains("VERTEBRATE")==false);
-	then
-		section.redactNot("NAME", 2, "Not Redacted because Section contains no Vertebrate");
-		section.redactNot("ADDRESS", 2, "Not Redacted because Section contains no Vertebrate");
-	end
-
-
-rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
-	when
-		eval(section.contains("VERTEBRATE")==true && section.contains("NO_REDACTION_INDICATOR")==true);
-	then
-		section.redactNot("NAME", 3, "Vertebrate was found, but also a no redaction indicator");
-		section.redactNot("ADDRESS", 3, "Vertebrate was found, but also a no redaction indicator");
-	end
-
-
-rule "4: Redact contact information, if applicant is found"
-	when
-		eval(section.getText().toLowerCase().contains("applicant"));
-	then
-		section.redactLineAfter("Name:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactBetween("Address:", "Contact", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Contact point:",  "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Phone:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Fax:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("E-mail:", "ADDRESS", 4, "Redacted because of Rule 4");
-	end
-
-
-rule "5: Redact contact information, if 'Producer of the plant protection product' is found"
-	when
-		eval(section.getText().contains("Producer of the plant protection product"));
-	then
-		section.redactLineAfter("Name:", "ADDRESS", 5, "xxxx");
-		section.redactBetween("Address:", "Contact", "ADDRESS", 5, "xxxx");
-		section.redactBetween("Contact:", "Phone", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("Phone:", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("Fax:", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("E-mail:", "ADDRESS", 5, "xxxx");
-	end

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DummyTest.java

@@ -1,14 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import org.junit.Test;
-
-/**
- *
- */
-public class DummyTest {
-
-    @Test
-    public void dummy(){
-        System.out.println("Hello World");
-    }
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -1,32 +1,222 @@
 package com.iqser.red.service.redaction.v1.server;
 
-
+import static org.mockito.Mockito.when;
 import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT;
 
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
+import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.iqser.red.service.configuration.v1.api.model.DefaultColor;
+import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
+import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
 @Ignore
 @RunWith(SpringRunner.class)
 @SpringBootTest(webEnvironment = DEFINED_PORT)
 public class RedactionIntegrationTest {
 
+    private static final String RULES = loadFromClassPath("drools/rules.drl");
+    private static final String VERTEBRATES_CODE = "vertebrate";
+    private static final String ADDRESS_CODE = "address";
+    private static final String NAME_CODE = "name";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
+    private static final String MUST_REDACT = "must_redact";
+
     @Autowired
     private RedactionController redactionController;
 
+    @MockBean
+    private RulesClient rulesClient;
+
+    @MockBean
+    private DictionaryClient dictionaryClient;
+
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, float[]> typeColorMap = new HashMap<>();
+    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
+    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
+
+    @TestConfiguration
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources()
+                    .newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+    }
+
+
+    @Before
+    public void stubClients() {
+
+        when(rulesClient.getVersion()).thenReturn(0L);
+        when(rulesClient.getRules()).thenReturn(new RulesResponse(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        when(dictionaryClient.getVersion()).thenReturn(0L);
+        when(dictionaryClient.getAllTypes()).thenReturn(TypeResponse.builder().types(getTypeResponse()).build());
+        when(dictionaryClient.getDictionaryForType(VERTEBRATES_CODE)).thenReturn(getDictionaryResponse(VERTEBRATES_CODE));
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
+        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT)).thenReturn(getDictionaryResponse(MUST_REDACT));
+        when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor(new float[]{1f, 0.502f, 0f}));
+    }
+
+
+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(NAME_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/names.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrates.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/addresses.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private void loadTypeForTest() {
+
+        typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0});
+        typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1});
+        typeColorMap.put(NAME_CODE, new float[]{1, 1, 0});
+        typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{0.8f, 0, 0.8f});
+        typeColorMap.put(REDACTION_INDICATOR, new float[]{1, 0.502f, 0.1f});
+        typeColorMap.put(HINT_ONLY, new float[]{0.8f, 1, 0.8f});
+        typeColorMap.put(MUST_REDACT, new float[]{1, 0, 0});
+
+        hintTypeMap.put(VERTEBRATES_CODE, true);
+        hintTypeMap.put(ADDRESS_CODE, false);
+        hintTypeMap.put(NAME_CODE, false);
+        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY, true);
+        hintTypeMap.put(MUST_REDACT, true);
+
+        caseInSensitiveMap.put(VERTEBRATES_CODE, true);
+        caseInSensitiveMap.put(ADDRESS_CODE, false);
+        caseInSensitiveMap.put(NAME_CODE, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY, true);
+        caseInSensitiveMap.put(MUST_REDACT, true);
+    }
+
+
+    private List<TypeResult> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> TypeResult.builder()
+                        .type(typeColor.getKey())
+                        .color(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+
+    private DictionaryResponse getDictionaryResponse(String type) {
+
+        return DictionaryResponse.builder()
+                .color(typeColorMap.get(type))
+                .entries(dictionary.get(type))
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .build();
+    }
 
 
     @Test
@@ -35,7 +225,9 @@ public class RedactionIntegrationTest {
         long start = System.currentTimeMillis();
         ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
 
-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
         request.setFlatRedaction(false);
 
         RedactionResult result = redactionController.redact(request);
@@ -49,13 +241,36 @@ public class RedactionIntegrationTest {
         System.out.println("numberOfPages: " + result.getNumberOfPages());
     }
 
+    @Test
+    public void testTableRedaction() throws IOException {
+
+        long start = System.currentTimeMillis();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
+
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
+
+        RedactionResult result = redactionController.redact(request);
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
+            fileOutputStream.write(result.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
 
     @Test
     public void classificationTest() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
 
-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
 
         RedactionResult result = redactionController.classify(request);
 
@@ -70,7 +285,9 @@ public class RedactionIntegrationTest {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
 
-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
 
         RedactionResult result = redactionController.sections(request);
 
@@ -79,12 +296,15 @@ public class RedactionIntegrationTest {
         }
     }
 
+
     @Test
     public void htmlTablesTest() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
 
-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
 
         RedactionResult result = redactionController.htmlTables(request);
 
@@ -93,12 +313,15 @@ public class RedactionIntegrationTest {
         }
     }
 
+
     @Test
     public void htmlTableRotationTest() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
 
-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
 
         RedactionResult result = redactionController.htmlTables(request);
 
@@ -107,4 +330,23 @@ public class RedactionIntegrationTest {
         }
     }
 
+
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java

@@ -0,0 +1,175 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.iqser.red.service.configuration.v1.api.model.DefaultColor;
+import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
+import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+public class EntityRedactionServiceTest {
+
+    private static final String DEFAULT_RULES = loadFromClassPath("drools/rules.drl");
+    private static final String NAME_CODE = "name";
+    private static final String ADDRESS_CODE = "address";
+
+    @MockBean
+    private DictionaryClient dictionaryClient;
+
+    @MockBean
+    private RulesClient rulesClient;
+
+    @Autowired
+    private EntityRedactionService entityRedactionService;
+
+    @Autowired
+    private PdfSegmentationService pdfSegmentationService;
+
+    @TestConfiguration
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            InputStream input = new ByteArrayInputStream(DEFAULT_RULES.getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources()
+                    .newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+    }
+
+
+    @Test
+    public void testNestedEntitiesRemoval() {
+
+        Set<Entity> entities = new HashSet<>();
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0);
+        entities.add(nested);
+        entities.add(nesting);
+        entityRedactionService.removeEntitiesContainedInLarger(entities);
+
+        assertThat(entities.size()).isEqualTo(1);
+        assertThat(entities).contains(nesting);
+
+    }
+
+
+    @Test
+    public void testTableRedaction() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
+
+        RedactionRequest redactionRequest = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
+
+        String tableRules = "package drools\n" +
+                "\n" +
+                "import com.iqser.red.service.redaction.v1.server.redaction.model.Section\n" +
+                "\n" +
+                "global Section section\n" +
+                "rule \"9: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study\"\n" +
+                "    when\n" +
+                "        Section(tabularData != null && tabularData.size() > 0\n" +
+                "            && tabularData.containsKey(\"Vertebrate study Y/N\")\n" +
+                "            && tabularData.get(\"Vertebrate study Y/N\").equals(\"Y\")\n" +
+                "        )\n" +
+                "    then\n" +
+                "        section.redact(\"name\", 9, \"Redacted because row is a vertebrate study\");\n" +
+                "        section.redact(\"address\", 9, \"Redacted because rows is a vertebrate study\");\n" +
+                "        section.highlightCell(\"Vertebrate study Y/N\", 9);\n" +
+                "    end";
+        when(rulesClient.getVersion()).thenReturn(1L);
+        when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));
+        TypeResponse typeResponse = TypeResponse.builder()
+                .types(Arrays.asList(
+                        TypeResult.builder().type(NAME_CODE).color(new float[]{1, 1, 0}).build(),
+                        TypeResult.builder().type(ADDRESS_CODE).color(new float[]{0, 1, 1}).build()))
+                .build();
+        when(dictionaryClient.getAllTypes()).thenReturn(typeResponse);
+        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
+                .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
+                .build();
+        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        DictionaryResponse addressResponse = DictionaryResponse.builder()
+                .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
+                .build();
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+        when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor());
+        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
+            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
+            entityRedactionService.processDocument(classifiedDoc);
+            assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
+            assertThat(classifiedDoc.getEntities().get(1)).hasSize(5); // 4 out of 5 entities recognized on page 1
+        }
+    }
+
+
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java

@@ -0,0 +1,64 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class ResourceLoader {
+
+    public Map<String, String> loadDictionaryFiles() {
+
+        String name = "dictionaries/";
+
+        List<String> files;
+        try {
+            files = IOUtils.readLines(ResourceLoader.class.getClassLoader().getResourceAsStream(name), "UTF-8");
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + name, e);
+        }
+        return files.stream().collect(Collectors.toMap(ResourceLoader::getFileName, s -> name + s));
+    }
+
+    private String getFileName(String filePath) {
+        return filePath.substring(0, filePath.indexOf(".txt"));
+    }
+
+    public Set<String> load(String classpathPath) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            return br.lines().collect(Collectors.toSet());
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
+        }
+    }
+
+    public String loadToString(String classpathPath) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            return br.lines().collect(Collectors.joining("\n"));
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class TextNormalizationUtilities {
+
+    /**
+     * Revert hyphenation due to line breaks.
+     * @param text Text to be processed.
+     * @return Text without line-break hyphenation.
+     */
+    public static String removeHyphenLineBreaks(String text) {
+        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java

@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+public class TextNormalizationUtilitiesTest {
+
+    @Test
+    public void testHyphenRemoval() {
+
+        String test = "Without these peo-\nple, this conference would not happen";
+        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
+                .contains("\npeople");
+
+        test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
+        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
+                .contains("\neinzelnen", "\nVertrages");
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt

@@ -0,0 +1,2 @@
+guideline
+unpublished

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt

@@ -0,0 +1,3 @@
+Batches Produced at
+CTL
+for determination of residues

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt

@@ -0,0 +1,3 @@
+published paper
+in vitro
+in-vitro

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt

@@ -0,0 +1,9 @@
+in vivo
+in-vivo
+dermal penetration
+oral toxicity
+oral-toxicity
+acute toxicity
+acute-toxicity
+eco toxicity
+eco-toxicity

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt

@@ -1,48 +1,63 @@
-Vulpes vulpes
+a. sylvaticus
 african clawed frog
 agalychnis callidryas
+albino rat
+american bullfrog tadpole
+american toad
 amphibian
 amphibians
-American bullfrog tadpole
-american toad
 anad platyrhynchos
-Anas platyrhynchos
+anas platyrhynchos
 anuran
 anurans
+apodemus
+apodemus flavicollis
 apodemus syl vaticus
+apodemus sylvaticus
+arvicola terrestris
 avian
+bank vole
 bird
 birds
 bluegill
 bluegill sunfish
 bobwhite
 bobwhite quail
-bullfrog
-Bufo americanus
 brachydanio rerio
+brown hare
+bufo americanus
+bullfrog
 canary
 carassius carassius
 carp
+catesbeiana
 catfish
 cattle
 cattles
 channel catfish
-Chinook
 chicken
-Colinus virginianus
+chinese hamster
+chinese hamsters
+chinook
+coho salmon
 colinus virginianus
-Common carp
+columba palumbus
+columbidae
+common carp
+common vole
 coturnix japonica
-Coturnix japonica
 cow
 cows
-Crucian carp
+crocidura russula
+crucian carp
 cyprinodon variegatus
 cyprinus carpio
 dog
 dogs
 duck
 ducks
+european brown hare
+european rabbit
 fathead minnow
 fish
 fishes
@@ -56,56 +71,84 @@ galaxias truttaceus
 gasterosteus aculeatus
 goat
 goats
+greater white-toothed shrew
 guinea
 guinea pig
 guinea pigs
-Guppy
+guinea-pigs
+guppy
 hamster
 hamsters
 hen
 hens
-Hyla versicolor
+house mouse
+hyla versicolor
 ictalurus melas
 ictalurus punctatus
 japanese quail
 japonica
+kisutch
+lagomorph
 lebistes reticulatus
 leiostomus xanthurus
 leisostomus xanthurus
 lepomis macrochirus
+lepus europaeus
+limnocharis
+limnodynastes
+limnodynastes tasmaniensis
 livestock
 livestocks
+mallard
 mallard duck
 mammal
+mammalian
 mammals
-Mammalian
+marten
+martes
 mice
+microtus
+microtus agrestis
+microtus arvalis
+microtus subterraneus
 midwestern anurans
+minnow
+minnows
 monkey
 mouse
+mus musculus
+myodes glareolus
 northern bobwhite
+o. cuniculus
 o. mykiss
-Oncorhynchus mykiss
-Oncorhynchus
-O. mykiss
+o. tshawytscha
+oncorhynchus
+oncorhynchus mykiss
+oncorhynchus tshawytscha
+oryctolagus cuniculus
 oryzias melastigma
 oryzias melastigma larvae
 p. promelas
 pagrus major
+palumbus
 pig
 pigeon
 pigeons
+pigs
+pimephales promela
 pimephales promelas
-Pseudacris triseriata
 poecilia reticulata
 poultry
+pseudacris
+pseudacris triseriata
 quail
+r. catesbeiana
 rabbit
 rabbits
 rainbow trout
-Rana limnocharis
 rana
-limnocharis
+rana catesbeiana
+rana limnocharis
 rana pipiens
 rat
 rats
@@ -114,120 +157,35 @@ reptiles
 ricefish
 ruminant
 ruminants
+salmo gairdneri
+salmon
+serinus canaria
 sheepshead minnow
 sheepshead minnows
-spea multiplicata
-Salmo gairdneri
-salmon
-spotted march frog
-tadpoles
-treefrog
-toad
-terrestrial vertrebrates
-Limnodynastes tasmaniensis
-trout
-Vulpes vulpes
-wistar
-xenopus laevis
-xenpous leavis
-zebra fish
-zebrafish
-Salmo gairdneri
-minnow
-minnows
-Pimephales promela
-Cyprinodon variegatus
-limnodynastes
-Rana catesbeiana
-R. catesbeiana
-coho salmon
-Oncorhynchus tshawytscha
-O. tshawytscha
-tshawytscha
-catesbeiana
-kisutch
-Pseudacris triseriata
-Pseudacris
-triseriata
-Wood pigeon
-Columba palumbus
-palumbus
-Columbidae
 shrew
 shrews
-bank vole
-common vole
+sorex araneus
+spea multiplicata
+spotted march frog
+tadpoles
+terrestrial vertrebrates
+toad
+treefrog
+triseriata
+trout
+tshawytscha
 vole
 voles
-lagomorph
-Wood mouse
-Apodemus sylvaticus
-A. sylvaticus
-Apodemus flavicollis
-Apodemus
-mus musculus
-Microtus arvalis
-Microtus agrestis
-Microtus
-Arvicola terrestris
-Sorex araneus
-Myodes glareolus
-yellow-necked mouse
-house mouse
-Oryctolagus cuniculus
-marten
-martes
+vulpes vulpes
+white rabbits
 white-toothed shrew
-greater white-toothed shrew
-Lepus europaeus
-brown hare
-European brown hare
-European rabbit
-O. cuniculus
-Crocidura russula
-Chinese Hamster
-Rat
-Rats
-Dog
-Chinese hamsters
-Chinese hamster
-Mouse
-Guinea pig
-Wistar rats
-Rabbit
-mammalian
-Japanese quail
-Microtus subterraneus
-Lepomis macrochirus
-P. promelas
-Cyprinus carpio
-Fish
-Ictalurus punctatus
-Carassius carassius
-Lepomis macrochirus
-Poecilia reticulata
-Lebistes reticulatus
-Lepomis macrochirus
-Leiostomus xanthurus
-Pimephales promelas
-Lepomis macrochirus
-Albino rat
-Hen
-Goat
-Livestock
-Guinea Pigs
-Hamster
+wistar
+wistar rats
+wood mice
 wood mouse
-Rabbits
-Mice
-Rainbow trout
-Canary
-Serinus canaria
-Guinea Pig
-Cow
-Pigs
-Poultry
-Guinea-pigs
-White rabbits
-Birds
-Wood mice
+wood pigeon
+xenopus laevis
+xenpous leavis
+yellow-necked mouse
+zebra fish
+zebrafish

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -0,0 +1,112 @@
+package drools
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+
+global Section section
+
+
+rule "1: Redacted because Section contains Vertebrate"
+	when
+		eval(section.contains("vertebrate")==true);
+	then
+		section.redact("name", 1, "Redacted because Section contains Vertebrate");
+		section.redact("address", 1, "Redacted because Section contains Vertebrate");
+	end
+
+
+rule "2: Not Redacted because Section contains no Vertebrate"
+	when
+		eval(section.contains("vertebrate")==false);
+	then
+		section.redactNot("name", 2, "Not Redacted because Section contains no Vertebrate");
+		section.redactNot("address", 2, "Not Redacted because Section contains no Vertebrate");
+	end
+
+
+rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
+	when
+		eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true);
+	then
+		section.redactNot("name", 3, "Vertebrate was found, but also a no redaction indicator");
+		section.redactNot("address", 3, "Vertebrate was found, but also a no redaction indicator");
+	end
+
+
+rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
+	when
+		eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true && section.contains("redaction_indicator")==true);
+	then
+		section.redact("name", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator");
+		section.redact("address", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator");
+	end
+
+
+rule "5: Do not redact in guideline sections"
+	when
+		eval(section.headlineContainsWord("guideline") || section.headlineContainsWord("Guidance"));
+	then
+		section.redactNot("name", 5, "Section is a guideline section.");
+		section.redactNot("address", 5, "Section is a guideline section.");
+	end
+
+rule "6: Redact if must redact entry is found"
+	when
+		eval(section.contains("must_redact")==true);
+	then
+		section.redact("name", 6, "must_redact entry was found.");
+		section.redact("address", 6, "must_redact entry was found.");
+	end
+
+
+rule "7: Redact contact information, if applicant is found"
+	when
+		eval(section.headlineContainsWord("applicant") || section.getText().contains("Applicant"));
+	then
+		section.redactLineAfter("Name:", "address", 7, "Applicant information was found");
+		section.redactBetween("Address:", "Contact", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Contact point:",  "address", 7, "Applicant information was found");
+		section.redactLineAfter("Phone:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Fax:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Tel.:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Tel:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("E-mail:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Email:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Contact:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Telephone number:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Fax number:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Telephone:", "address", 7, "Applicant information was found");
+		section.redactBetween("No:", "Fax", "address", 7, "Applicant information was found");
+		section.redactBetween("Contact:", "Tel.:", "address", 7, "Applicant information was found");
+	end
+
+rule "8: Redact contact information, if Producer is found"
+	when
+		eval(section.getText().toLowerCase().contains("producer of the plant protection") || section.getText().toLowerCase().contains("producer of the active substance") || section.getText().contains("Manufacturer of the active substance") || section.getText().contains("Manufacturer:") || section.getText().contains("Producer or producers of the active substance"));
+	then
+		section.redactLineAfter("Name:", "address", 8, "Producer was found");
+		section.redactBetween("Address:", "Contact", "address", 8, "Producer was found");
+		section.redactBetween("Contact:", "Phone", "address", 8, "Producer was found");
+		section.redactBetween("Contact:", "Telephone number:", "address", 8, "Producer was found");
+		section.redactBetween("Address:", "Manufacturing", "address", 8, "Producer was found");
+		section.redactLineAfter("Telephone:", "address", 8, "Producer was found");
+		section.redactLineAfter("Phone:", "address", 8, "Producer was found");
+		section.redactLineAfter("Fax:", "address", 8, "Producer was found");
+		section.redactLineAfter("E-mail:", "address", 8, "Producer was found");
+		section.redactLineAfter("Contact:", "address", 8, "Producer was found");
+		section.redactLineAfter("Fax number:", "address", 8, "Producer was found");
+		section.redactLineAfter("Telephone number:", "address", 8, "Producer was found");
+		section.redactLineAfter("Tel:", "address", 8, "Producer was found");
+		section.redactBetween("No:", "Fax", "address", 8, "Producer was found");
+	end
+
+rule "9: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study"
+    when
+        Section(tabularData != null && tabularData.size() > 0
+            && tabularData.containsKey("Vertebrate study Y/N")
+            && tabularData.get("Vertebrate study Y/N").equals("Y")
+        )
+    then
+        section.redact("name", 9, "Redacted because row is a vertebrate study");
+        section.redact("address", 9, "Redacted because rows is a vertebrate study");
+        section.highlightCell("Vertebrate study Y/N", 9);
+    end