Do not remove images at reanalysis

Merge in RED/redaction-service from RED-1276 to master

* commit '8375c04829347f43c2151e80dd4f4571b0c2757d':
  RED-1276: Fixed annotation position problem for 270° rotated pages

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -72,14 +72,6 @@ public class PlanSpec {
                         .jobs(new Job("Default Job",
                                 new BambooKey("JOB1"))
                                 .tasks(
-                                        new ScriptTask()
-                                                .description("Clean")
-                                                .inlineBody("#!/bin/bash\n" +
-                                                        "set -e\n" +
-                                                        "rm -rf ./*"),
-                                        new VcsCheckoutTask()
-                                                .description("Checkout Default Repository")
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
                                         new ScriptTask()
                                                 .description("Clean")
                                                 .inlineBody("#!/bin/bash\n" +

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.0.2</version>
+        <version>1.0.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.1.7</version>
+                <version>1.2.5</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java

@@ -14,5 +14,6 @@ public class AnalyzeResult {
     private int numberOfPages;
     private RedactionLog redactionLog;
     private SectionGrid sectionGrid;
+    private Text text;
 
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class ManualForceRedact {
+
+    private String id;
+    private String user;
+    private Status status;
+    private String legalBasis;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java

@@ -1,5 +1,5 @@
 package com.iqser.red.service.redaction.v1.model;
 
 public enum ManualRedactionType {
-    ADD, REMOVE
+    ADD, REMOVE, FORCE_REDACT
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java

@@ -20,6 +20,9 @@ public class ManualRedactions {
     @Builder.Default
     private Set<IdRemoval> idsToRemove = new HashSet<>();
 
+    @Builder.Default
+    private Set<ManualForceRedact> forceRedacts = new HashSet<>();
+
     @Builder.Default
     private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();
 

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ReanalyzeResult.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ReanalyzeResult {
+
+    private RedactionLog redactionLog;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -40,4 +40,7 @@ public class RedactionLogEntry {
     @Builder.Default
     private List<Comment> comments = new ArrayList<>();
 
+    private int startOffset;
+    private int endOffset;
+
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RenalyzeRequest.java

@@ -0,0 +1,22 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.time.OffsetDateTime;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class RenalyzeRequest {
+
+    private byte[] document;
+    private String ruleSetId;
+    private ManualRedactions manualRedactions;
+    private Text text;
+    private RedactionLog redactionLog;
+    private OffsetDateTime lastProcessed;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java

@@ -0,0 +1,33 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@RequiredArgsConstructor
+@NoArgsConstructor
+@AllArgsConstructor
+public class SectionArea {
+
+    @NonNull
+    private Point topLeft;
+
+    @NonNull
+    private float width;
+
+    @NonNull
+    private float height;
+
+    @NonNull
+    private int page;
+
+    private String header;
+
+    public boolean contains(Rectangle other) {
+        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY()  && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight();
+    }
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionText.java

@@ -0,0 +1,25 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SectionText {
+
+    private int sectionNumber;
+    private String text;
+
+    private boolean isTable;
+    private String headline;
+
+    private List<SectionArea> sectionAreas = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Text.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class Text {
+
+    private List<SectionText> sectionTexts = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -4,8 +4,11 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.model.ReanalyzeResult;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.model.RenalyzeRequest;
+
 import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -21,6 +24,9 @@ public interface RedactionResource {
     @PostMapping(value = "/analyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
     AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest);
 
+    @PostMapping(value = "/reanalyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
+    ReanalyzeResult reanalyze(@RequestBody RenalyzeRequest renalyzeRequest);
+
     @PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
     AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
 

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -20,7 +20,7 @@
         <dependency>
             <groupId>com.iqser.red.service</groupId>
             <artifactId>configuration-service-api-v1</artifactId>
-            <version>2.0.0</version>
+            <version>2.2.9</version>
         </dependency>
         <dependency>
             <groupId>org.drools</groupId>
@@ -73,11 +73,7 @@
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-starter-openfeign</artifactId>
         </dependency>
-        <!-- ribbon must be included because transitive a dependency on it.-->
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-starter-netflix-ribbon</artifactId>
-        </dependency>
+
         <!-- test dependencies -->
         <dependency>
             <groupId>org.springframework.boot</groupId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -1,10 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import org.springframework.boot.SpringApplication;
-import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
@@ -12,10 +8,14 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Import;
 
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
 @Import({DefaultWebMvcConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
-@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
+@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {
 
     public static void main(String[] args) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -8,6 +8,7 @@ import java.util.Map;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.model.SectionText;
 
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -20,6 +21,7 @@ public class Document {
     private List<Paragraph> paragraphs = new ArrayList<>();
     private List<Header> headers = new ArrayList<>();
     private List<Footer> footers = new ArrayList<>();
+    private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
     private Map<Integer, List<Entity>> entities = new HashMap<>();
     private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
     private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
@@ -31,4 +33,6 @@ public class Document {
     private SectionGrid sectionGrid = new SectionGrid();
     private long dictionaryVersion;
     private long rulesVersion;
+
+    private List<SectionText> sectionText = new ArrayList<>();
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/UnclassifiedText.java

@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class UnclassifiedText {
+
+    private List<TextBlock> textBlocks;
+
+
+    public SearchableText getSearchableText() {
+
+        SearchableText searchableText = new SearchableText();
+        textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
+        return searchableText;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -111,6 +111,8 @@ public class ClassificationService {
             textBlock.setClassification("TextBlock Italic");
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
             textBlock.setClassification("TextBlock Unknown");
+        } else {
+            textBlock.setClassification("Other");
         }
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -4,11 +4,14 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.model.ReanalyzeResult;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.model.RenalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.model.Text;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
@@ -17,13 +20,16 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationSer
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -46,6 +52,7 @@ public class RedactionController implements RedactionResource {
     private final DroolsExecutionService droolsExecutionService;
     private final DictionaryService dictionaryService;
     private final AnnotationService annotationService;
+    private final ReanalyzeService reanalyzeService;
 
 
     @Override
@@ -67,8 +74,9 @@ public class RedactionController implements RedactionResource {
             return AnalyzeResult.builder()
                     .sectionGrid(classifiedDoc.getSectionGrid())
                     .redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
-                    .getRulesVersion(), analyzeRequest.getRuleSetId()))
+                            .getRulesVersion(), analyzeRequest.getRuleSetId()))
                     .numberOfPages(classifiedDoc.getPages().size())
+                    .text(new Text(classifiedDoc.getSectionText()))
                     .build();
 
         } catch (Exception e) {
@@ -78,6 +86,12 @@ public class RedactionController implements RedactionResource {
     }
 
 
+    public ReanalyzeResult reanalyze(@RequestBody RenalyzeRequest renalyzeRequest) {
+
+        return reanalyzeService.reanalyze(renalyzeRequest);
+    }
+
+
     public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(annotateRequest.getDocument()))) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFAreaTextStripper.java

@@ -0,0 +1,83 @@
+package com.iqser.red.service.redaction.v1.server.parsing;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pdfbox.text.PDFTextStripperByArea;
+import org.apache.pdfbox.text.TextPosition;
+
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.Getter;
+import lombok.Setter;
+
+public class PDFAreaTextStripper extends PDFTextStripperByArea {
+
+    @Getter
+    private List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+
+    @Setter
+    private int pageNumber;
+
+    public PDFAreaTextStripper() throws IOException {
+
+    }
+
+    @Override
+    public void writeString(String text, List<TextPosition> textPositions) throws IOException {
+
+        int startIndex = 0;
+        for (int i = 0; i <= textPositions.size() - 1; i++) {
+
+            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
+                    .getUnicode()
+                    .equals("\u00A0"))) {
+                startIndex++;
+                continue;
+            }
+
+            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
+            if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                }
+                startIndex = i;
+            }
+
+            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
+                    .getUnicode()
+                    .equals("\u00A0")) && i <= textPositions.size() - 2) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                }
+                startIndex = i + 1;
+            }
+        }
+
+        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
+        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
+                .getUnicode()
+                .equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
+            sublist = sublist.subList(0, sublist.size() - 1);
+        }
+        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
+                .getUnicode()
+                .equals("\u00A0")))) {
+            textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+        }
+        super.writeString(text);
+    }
+
+
+    public void clearPositions(){
+        textPositionSequences = new ArrayList<>();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -101,10 +101,13 @@ public class TextPositionSequence implements CharSequence {
         }
     }
 
+
     public float getRotationAdjustedY() {
+
         return textPositions.get(0).getY();
     }
 
+
     public float getY1() {
 
         if (textPositions.get(0).getRotation() == 90) {
@@ -202,6 +205,13 @@ public class TextPositionSequence implements CharSequence {
             posXEnd = textPositions.get(0).getYDirAdj() + 2;
             posYInit = getY1();
             posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
+        } else if (textPositions.get(0).getRotation() == 270) {
+            posYInit = textPositions.get(0).getPageHeight() - getX1();
+            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
+                    .getWidth() - textPositions.get(textPositions.size() - 1).getWidth() - 1;
+            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + height;
         } else {
             posXEnd = textPositions.get(textPositions.size() - 1)
                     .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
@@ -213,5 +223,4 @@ public class TextPositionSequence implements CharSequence {
         return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrement.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.Set;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class DictionaryIncrement {
+
+    private Set<DictionaryIncrementValue> values;
+    private long dictionaryVersion;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrementValue.java

@@ -0,0 +1,13 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class DictionaryIncrementValue {
+
+    private String value;
+    private boolean caseinsensitive;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java

@@ -3,6 +3,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 
 import java.io.Serializable;
 import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -17,11 +20,12 @@ public class DictionaryModel implements Serializable {
     private boolean caseInsensitive;
     private boolean hint;
     private boolean recommendation;
-    private Set<String> entries;
+    private Set<DictionaryEntry> entries;
     private Set<String> localEntries;
 
     public Set<String> getValues(boolean local){
-        return local ? localEntries : entries;
+        return local ? localEntries : entries.stream().filter(e -> !e.isDeleted()).map(e-> e.getValue()).collect(Collectors
+                .toSet());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -39,7 +39,7 @@ public class Entity {
     private String textAfter;
 
 
-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter) {
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end) {
 
         this.word = word;
         this.type = type;
@@ -53,6 +53,8 @@ public class Entity {
         this.isDictionaryEntry = isDictionaryEntry;
         this.textBefore = textBefore;
         this.textAfter = textAfter;
+        this.start = start;
+        this.end = end;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ReanalysisSection.java

@@ -0,0 +1,34 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class ReanalysisSection {
+
+    private int sectionNumber;
+    private String headline;
+    private List<TextBlock> textBlocks;
+    private Map<String, CellValue> tabularData = new HashMap<>();
+    private List<Integer> cellStarts;
+
+
+    public SearchableText getSearchableText() {
+
+        SearchableText searchableText = new SearchableText();
+        textBlocks.forEach(block -> {
+            if (block instanceof TextBlock) {
+                searchableText.addAll(block.getSequences());
+            }
+        });
+        return searchableText;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -140,6 +140,48 @@ public class Section {
     }
 
 
+    public void expandToHintAnnotationByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group, String asType) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Set<Entity> expanded = new HashSet<>();
+        for (Entity entity : entities) {
+
+            if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
+                continue;
+            }
+
+            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+
+            while (matcher.find()) {
+                String match = matcher.group(group);
+                if (StringUtils.isNotBlank(match)) {
+                    expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null));
+                }
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+    }
+
+
+    public void addHintAnnotationByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Matcher matcher = compiledPattern.matcher(searchText);
+
+        while (matcher.find()) {
+            String match = matcher.group(group);
+            if (StringUtils.isNotBlank(match)) {
+                Set<Entity> found = findEntities(match.trim(), asType, false, false, 0, null, null);
+                EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+            }
+        }
+    }
+
+
     public void redactIfPrecededBy(String prefix, String type, int ruleNumber, String reason, String legalBasis) {
 
         entities.forEach(entity -> {
@@ -160,6 +202,13 @@ public class Section {
     }
 
 
+    public void addRedaction(String value, String asType, int ruleNumber, String reason, String legalBasis) {
+
+        Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis);
+        EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
+    }
+
+
     public void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
                                 String legalBasis) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -1,42 +1,46 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.configuration.v1.api.model.Colors;
-import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
-import com.iqser.red.service.configuration.v1.api.model.TypeResult;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
-import feign.FeignException;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.SerializationUtils;
-import org.springframework.stereotype.Service;
-
 import java.awt.Color;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.SerializationUtils;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.configuration.v1.api.model.Colors;
+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
+
+import feign.FeignException;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class DictionaryService {
 
-
     private final DictionaryClient dictionaryClient;
 
-
     private Map<String, DictionaryRepresentation> dictionariesByRuleSets = new HashMap<>();
 
 
-    public void updateDictionary(String ruleSetId) {
+    public long updateDictionary(String ruleSetId) {
 
         long version = dictionaryClient.getVersion(ruleSetId);
 
@@ -45,6 +49,26 @@ public class DictionaryService {
         if (foundDictionary == null || version > foundDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(ruleSetId, version);
         }
+
+        return version;
+    }
+
+
+    public DictionaryIncrement getDictionaryIncrements(String ruleSetId, long fromVersion) {
+
+        long version = updateDictionary(ruleSetId);
+
+        Set<DictionaryIncrementValue> newValues = new HashSet<>();
+        List<DictionaryModel> dictionaryModels = dictionariesByRuleSets.get(ruleSetId).getDictionary();
+        dictionaryModels.forEach(dictionaryModel -> {
+            dictionaryModel.getEntries().forEach(dictionaryEntry -> {
+                if (dictionaryEntry.getVersion() > fromVersion) {
+                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                }
+            });
+        });
+
+        return new DictionaryIncrement(newValues, version);
     }
 
 
@@ -63,7 +87,6 @@ public class DictionaryService {
                         .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                         .collect(Collectors.toList());
 
-
                 dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
 
                 Colors colors = dictionaryClient.getColors(ruleSetId);
@@ -86,6 +109,7 @@ public class DictionaryService {
 
 
     public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) {
+
         dictionary.getDictionaryModels().forEach(dm -> {
             if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
                 dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false);
@@ -98,17 +122,15 @@ public class DictionaryService {
     }
 
 
-    private Set<String> convertEntries(TypeResult t) {
+    private Set<DictionaryEntry> convertEntries(TypeResult t) {
+
+        Set<DictionaryEntry> entries = new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
+                .getEntries());
 
         if (t.isCaseInsensitive()) {
-            return dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
-                    .getEntries()
-                    .stream()
-                    .map(String::toLowerCase)
-                    .collect(Collectors.toSet());
-        } else {
-            return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId()).getEntries());
+            entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
         }
+        return entries;
     }
 
 
@@ -148,6 +170,7 @@ public class DictionaryService {
         return false;
     }
 
+
     public boolean isRecommendation(String type, String ruleSetId) {
 
         DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
@@ -159,6 +182,7 @@ public class DictionaryService {
 
 
     public Dictionary getDeepCopyDictionary(String ruleSetId) {
+
         List<DictionaryModel> copy = new ArrayList<>();
 
         var representation = dictionariesByRuleSets.get(ruleSetId);
@@ -170,15 +194,22 @@ public class DictionaryService {
         return new Dictionary(copy, representation.getDictionaryVersion());
     }
 
+
     public float[] getRequestRemoveColor(String ruleSetId) {
+
         return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
     }
 
+
     public float[] getNotRedactedColor(String ruleSetId) {
+
         return dictionariesByRuleSets.get(ruleSetId).getNotRedactedColor();
     }
 
+
     public float[] getRequestAddColor(String ruleSetId) {
+
         return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -17,12 +17,15 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
@@ -31,6 +34,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -62,7 +66,7 @@ public class EntityRedactionService {
 
             Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
             documentEntities.stream().forEach(entity -> {
-                if (dictionary.isHint(entity.getType())) {
+                if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
                     hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>())
                             .add(entity);
                 }
@@ -85,7 +89,7 @@ public class EntityRedactionService {
                         .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
                         .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                 .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
-                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter()));
+                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity.getStart(), entity.getEnd()));
             }
         }
 
@@ -109,24 +113,29 @@ public class EntityRedactionService {
             List<Table> tables = paragraph.getTables();
             for (Table table : tables) {
                 if (table.getColCount() == 2) {
-                    sectionSearchableTextPairs.addAll(processTableAsOneText(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+                    sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
                 } else {
-                    sectionSearchableTextPairs.addAll(processTablePerRow(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+                    sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
                 }
                 sectionNumber.incrementAndGet();
             }
-            sectionSearchableTextPairs.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph
+            sectionSearchableTextPairs.add(processText(classifiedDoc, paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph
                     .getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
             sectionNumber.incrementAndGet();
         }
 
         for (Header header : classifiedDoc.getHeaders()) {
-            sectionSearchableTextPairs.add(processText(header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionSearchableTextPairs.add(processText(classifiedDoc, header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
             sectionNumber.incrementAndGet();
         }
 
         for (Footer footer : classifiedDoc.getFooters()) {
-            sectionSearchableTextPairs.add(processText(footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionSearchableTextPairs.add(processText(classifiedDoc, footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionNumber.incrementAndGet();
+        }
+
+        for (UnclassifiedText unclassifiedText : classifiedDoc.getUnclassifiedTexts()) {
+            sectionSearchableTextPairs.add(processText(classifiedDoc, unclassifiedText.getSearchableText(), unclassifiedText.getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
             sectionNumber.incrementAndGet();
         }
 
@@ -163,7 +172,7 @@ public class EntityRedactionService {
     }
 
 
-    private List<SectionSearchableTextPair> processTablePerRow(Table table, ManualRedactions manualRedactions,
+    private List<SectionSearchableTextPair> processTablePerRow(Document classifiedDoc, Table table, ManualRedactions manualRedactions,
                                                                AtomicInteger sectionNumber, Dictionary dictionary,
                                                                boolean local,
                                                                Map<Integer, Set<Entity>> hintsPerSectionNumber) {
@@ -175,10 +184,17 @@ public class EntityRedactionService {
             Map<String, CellValue> tabularData = new HashMap<>();
             int start = 0;
             List<Integer> cellStarts = new ArrayList<>();
+            SectionText sectionText = new SectionText();
             for (Cell cell : row) {
+
                 if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
                     continue;
                 }
+
+                SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
+                        .getWidth(), (float) cell.getHeight(), cell.getTextBlocks().get(0).getSequences().get(0).getPage());
+                sectionText.getSectionAreas().add(sectionArea);
+
                 addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
                 int cellStart = start;
 
@@ -190,6 +206,7 @@ public class EntityRedactionService {
                                 .replaceAll("\n", "")
                                 .replaceAll(" ", "")
                                 .replaceAll("-", "");
+                        sectionArea.setHeader(headerName);
                         tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
                     });
                 }
@@ -200,6 +217,7 @@ public class EntityRedactionService {
                 }
                 cellStarts.add(cellStart);
                 start = start + cell.toString().trim().length() + 1;
+
             }
             Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
             surroundingWordsService.addSurroundingText(rowEntities, searchableRow, dictionary, cellStarts);
@@ -219,6 +237,14 @@ public class EntityRedactionService {
                     .dictionary(dictionary)
                     .build(), searchableRow));
 
+            if(!local) {
+                sectionText.setText(searchableRow.toString());
+                sectionText.setHeadline(table.getHeadline());
+                sectionText.setSectionNumber(sectionNumber.intValue());
+                sectionText.setTable(true);
+                classifiedDoc.getSectionText().add(sectionText);
+            }
+
             sectionNumber.incrementAndGet();
         }
 
@@ -226,18 +252,26 @@ public class EntityRedactionService {
     }
 
 
-    private List<SectionSearchableTextPair> processTableAsOneText(Table table, ManualRedactions manualRedactions,
+    private List<SectionSearchableTextPair> processTableAsOneText(Document classifiedDoc, Table table, ManualRedactions manualRedactions,
                                                                   AtomicInteger sectionNumber, Dictionary dictionary,
                                                                   boolean local,
                                                                   Map<Integer, Set<Entity>> hintsPerSectionNumber) {
 
         List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
         SearchableText entireTableText = new SearchableText();
+        SectionText sectionText = new SectionText();
         for (List<Cell> row : table.getRows()) {
             for (Cell cell : row) {
                 if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
                     continue;
                 }
+
+                if(!local) {
+                    SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
+                            .getWidth(), (float) cell.getHeight(), cell.getTextBlocks().get(0).getSequences().get(0).getPage());
+                    sectionText.getSectionAreas().add(sectionArea);
+                }
+
                 for (TextBlock textBlock : cell.getTextBlocks()) {
                     entireTableText.addAll(textBlock.getSequences());
                 }
@@ -245,6 +279,7 @@ public class EntityRedactionService {
             }
         }
 
+
         Set<Entity> rowEntities = findEntities(entireTableText, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
         surroundingWordsService.addSurroundingText(rowEntities, entireTableText, dictionary);
 
@@ -262,15 +297,38 @@ public class EntityRedactionService {
                 .dictionary(dictionary)
                 .build(), entireTableText));
 
+        if(!local) {
+            sectionText.setText(entireTableText.toString());
+            sectionText.setHeadline(table.getHeadline());
+            sectionText.setSectionNumber(sectionNumber.intValue());
+            sectionText.setTable(true);
+            classifiedDoc.getSectionText().add(sectionText);
+        }
+
         return sectionSearchableTextPairs;
     }
 
 
-    private SectionSearchableTextPair processText(SearchableText searchableText, List<TextBlock> paragraphTextBlocks,
+    private SectionSearchableTextPair processText(Document classifiedDoc, SearchableText searchableText, List<TextBlock> paragraphTextBlocks,
                                                   String headline, ManualRedactions manualRedactions,
                                                   AtomicInteger sectionNumber, Dictionary dictionary, boolean local,
                                                   Map<Integer, Set<Entity>> hintsPerSectionNumber) {
 
+        if(!local) {
+            SectionText sectionText = new SectionText();
+            for (TextBlock paragraphTextBlock : paragraphTextBlocks) {
+                SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock
+                        .getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage());
+                sectionText.getSectionAreas().add(sectionArea);
+            }
+
+            sectionText.setText(searchableText.toString());
+            sectionText.setHeadline(headline);
+            sectionText.setSectionNumber(sectionNumber.intValue());
+            sectionText.setTable(false);
+            classifiedDoc.getSectionText().add(sectionText);
+        }
+
         addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue());
         Set<Entity> entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local);
         surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);
@@ -291,7 +349,7 @@ public class EntityRedactionService {
     }
 
 
-    private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
+    public Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
                                      Dictionary dictionary, boolean local) {
 
         Set<Entity> found = new HashSet<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java

@@ -0,0 +1,280 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.awt.geom.Rectangle2D;
+import java.io.ByteArrayInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import com.iqser.red.service.redaction.v1.model.Comment;
+import com.iqser.red.service.redaction.v1.model.IdRemoval;
+import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
+import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.ReanalyzeResult;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RenalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.parsing.PDFAreaTextStripper;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ReanalysisSection;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class ReanalyzeService {
+
+    private final DictionaryService dictionaryService;
+    private final DroolsExecutionService droolsExecutionService;
+    private final SurroundingWordsService surroundingWordsService;
+    private final EntityRedactionService entityRedactionService;
+    private final RedactionLogCreatorService redactionLogCreatorService;
+
+
+    public ReanalyzeResult reanalyze(@RequestBody RenalyzeRequest renalyzeRequest) {
+
+        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(renalyzeRequest.getRuleSetId(), renalyzeRequest
+                .getRedactionLog()
+                .getDictionaryVersion());
+
+        Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(renalyzeRequest.getManualRedactions());
+        Map<String, List<Comment>> comments = null;
+        Set<ManualRedactionEntry> manualAdds = null;
+
+        if (renalyzeRequest.getManualRedactions() != null) {
+            // TODO comments will be removed from redactionLog, so we ignore this first.
+            comments = renalyzeRequest.getManualRedactions().getComments();
+            manualAdds = renalyzeRequest.getManualRedactions().getEntriesToAdd();
+        }
+
+        Set<Integer> sectionsToReanaylse = new HashSet<>();
+        for (RedactionLogEntry entry : renalyzeRequest.getRedactionLog().getRedactionLogEntry()) {
+            if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) {
+                sectionsToReanaylse.add(entry.getSectionNumber());
+            }
+        }
+
+        for (SectionText sectionText : renalyzeRequest.getText().getSectionTexts()) {
+
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrement.getValues())) {
+                sectionsToReanaylse.add(sectionText.getSectionNumber());
+            }
+
+            if (manualAdds != null) {
+                for (SectionArea sectionArea : sectionText.getSectionAreas()) {
+                    for (ManualRedactionEntry manualAdd : manualAdds) {
+                        for (Rectangle manualPosition : manualAdd.getPositions()) {
+                            if (sectionArea.contains(manualPosition)) {
+                                manualAdd.setSection(sectionText.getHeadline());
+                                manualAdd.setSectionNumber(sectionText.getSectionNumber());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        if (sectionsToReanaylse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) {
+            renalyzeRequest.getRedactionLog().setDictionaryVersion(dictionaryIncrement.getDictionaryVersion());
+            return ReanalyzeResult.builder().redactionLog(renalyzeRequest.getRedactionLog()).build();
+        }
+
+        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(renalyzeRequest.getDocument()))) {
+
+            List<ReanalysisSection> reanalysisSections = new ArrayList<>();
+            for (SectionText sectionText : renalyzeRequest.getText().getSectionTexts()) {
+
+                if (!sectionsToReanaylse.contains(sectionText.getSectionNumber())) {
+                    continue;
+                }
+
+                ReanalysisSection reanalysisSection = new ReanalysisSection();
+                reanalysisSection.setHeadline(sectionText.getHeadline());
+                reanalysisSection.setSectionNumber(sectionText.getSectionNumber());
+                List<TextBlock> textBlocks = new ArrayList<>();
+
+                Map<Integer, List<SectionArea>> sectionAreasPerPage = new HashMap<>();
+                for (SectionArea sectionArea : sectionText.getSectionAreas()) {
+                    sectionAreasPerPage.computeIfAbsent(sectionArea.getPage(), (x) -> new ArrayList<>())
+                            .add(sectionArea);
+                }
+
+                Map<String, CellValue> tabularData = new HashMap<>();
+                List<Integer> cellStarts = new ArrayList<>();
+                for (Integer page : sectionAreasPerPage.keySet()) {
+                    List<SectionArea> areasOnPage = sectionAreasPerPage.get(page);
+
+                    PDPage pdPage = pdDocument.getPage(page - 1);
+                    PDRectangle cropBox = pdPage.getCropBox();
+                    PDFAreaTextStripper textStripper = new PDFAreaTextStripper();
+                    textStripper.setPageNumber(page);
+
+                    int cellStart = 0;
+                    for (SectionArea sectionArea : areasOnPage) {
+
+                        Rectangle2D rect = null;
+                        if (pdPage.getRotation() == 90) {
+                            rect = new Rectangle2D.Float(sectionArea.getTopLeft().getY(), sectionArea.getTopLeft()
+                                    .getX(), sectionArea.getHeight(), sectionArea.getWidth() + 0.001f);
+                        } else {
+                            rect = new Rectangle2D.Float(sectionArea.getTopLeft().getX(), -sectionArea.getTopLeft()
+                                    .getY() + cropBox.getUpperRightY() - sectionArea.getHeight(), sectionArea.getWidth(), sectionArea
+                                    .getHeight() + 0.001f);
+                        }
+
+                        textStripper.addRegion(String.valueOf(1), rect);
+                        textStripper.extractRegions(pdPage);
+                        textStripper.getTextForRegion(String.valueOf(1));
+                        List<TextPositionSequence> positions = textStripper.getTextPositionSequences();
+
+                        TextBlock textBlock = new TextBlock(sectionArea.getTopLeft().getX(), sectionArea.getTopLeft()
+                                .getX() + sectionArea.getWidth(), sectionArea.getTopLeft()
+                                .getY(), sectionArea.getTopLeft().getY() + sectionArea.getHeight(), positions, 0);
+
+                        if (sectionText.isTable()) {
+                            Cell cell = new Cell();
+                            cell.addTextBlock(textBlock);
+                            tabularData.put(sectionArea.getHeader(), new CellValue(cell.getTextBlocks(), cellStart));
+                            cellStarts.add(cellStart);
+                            cellStart = cellStart + cell.toString().trim().length() + 1;
+                        }
+
+                        textBlocks.add(textBlock);
+                        textStripper.clearPositions();
+                    }
+
+                }
+                reanalysisSection.setTextBlocks(textBlocks);
+                reanalysisSection.setTabularData(tabularData);
+                reanalysisSections.add(reanalysisSection);
+                if (sectionText.isTable()) {
+                    reanalysisSection.setCellStarts(cellStarts);
+                }
+            }
+
+            //--
+
+            KieContainer kieContainer = droolsExecutionService.updateRules(renalyzeRequest.getRuleSetId());
+
+            Dictionary dictionary = dictionaryService.getDeepCopyDictionary(renalyzeRequest.getRuleSetId());
+
+            List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
+            for (ReanalysisSection reanalysisSection : reanalysisSections) {
+
+                Set<Entity> entities = entityRedactionService.findEntities(reanalysisSection.getSearchableText(), reanalysisSection
+                        .getHeadline(), reanalysisSection.getSectionNumber(), dictionary, false);
+                if (reanalysisSection.getCellStarts() != null) {
+                    surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection
+                            .getCellStarts());
+                } else {
+                    surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
+                }
+
+                sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
+                        .isLocal(false)
+                        .dictionaryTypes(dictionary.getTypes())
+                        .entities(entities)
+                        .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
+                        .searchText(reanalysisSection.getSearchableText().toString())
+                        .headline(reanalysisSection.getHeadline())
+                        .sectionNumber(reanalysisSection.getSectionNumber())
+                        .tabularData(reanalysisSection.getTabularData())
+                        .searchableText(reanalysisSection.getSearchableText())
+                        .dictionary(dictionary)
+                        .build(), reanalysisSection.getSearchableText()));
+            }
+
+            Set<Entity> entities = new HashSet<>();
+            sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
+                Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair
+                        .getSection());
+                entities.addAll(analysedRowSection.getEntities());
+            });
+
+            Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
+            for (Entity entity : entities) {
+                Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
+                for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
+                    sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
+                            .add(entityPositionSequence);
+                }
+
+                for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
+                    entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
+                            .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
+                                    .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
+                                    .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
+                                    .getStart(), entity.getEnd()));
+                }
+            }
+
+            List<RedactionLogEntry> newRedactionLogEntries = new ArrayList<>();
+            for (int page = 1; page <= pdDocument.getNumberOfPages(); page++) {
+                if (entitiesPerPage.get(page) != null) {
+                    newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, renalyzeRequest
+                            .getManualRedactions(), page, renalyzeRequest.getRuleSetId()));
+                }
+                newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, renalyzeRequest
+                        .getRuleSetId()));
+            }
+
+            Iterator<RedactionLogEntry> itty = renalyzeRequest.getRedactionLog().getRedactionLogEntry().iterator();
+            while (itty.hasNext()) {
+                RedactionLogEntry entry = itty.next();
+                if (sectionsToReanaylse.contains(entry.getSectionNumber()) && !entry.getType().equals("image") || entry.getSectionNumber() == 0) {
+                    itty.remove();
+                }
+            }
+
+            renalyzeRequest.getRedactionLog().getRedactionLogEntry().addAll(newRedactionLogEntries);
+            renalyzeRequest.getRedactionLog().setDictionaryVersion(dictionaryIncrement.getDictionaryVersion());
+
+            return ReanalyzeResult.builder().redactionLog(renalyzeRequest.getRedactionLog()).build();
+
+        } catch (Exception e) {
+            throw new RedactionException(e);
+        }
+
+    }
+
+
+    private Set<String> getForceAndRemoveIds(ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null) {
+            return new HashSet<>();
+        }
+
+        return Stream.concat(manualRedactions.getIdsToRemove()
+                .stream()
+                .map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId))
+                .collect(Collectors.toSet());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -4,6 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -14,6 +15,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Comment;
 import com.iqser.red.service.redaction.v1.model.IdRemoval;
+import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
@@ -54,11 +56,11 @@ public class RedactionLogCreatorService {
             addSectionGrid(classifiedDoc, page);
 
             if (classifiedDoc.getEntities().get(page) != null) {
-                addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
+                classifiedDoc.getRedactionLogEntities().addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, ruleSetId));
             }
 
             if (manualRedactionPages.contains(page)) {
-                addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
+                classifiedDoc.getRedactionLogEntities().addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, ruleSetId));
             }
 
             if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
@@ -105,13 +107,15 @@ public class RedactionLogCreatorService {
     }
 
 
-    private void addEntries(Document classifiedDoc, ManualRedactions manualRedactions, int page, String ruleSetId) {
+    public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions, int page, String ruleSetId) {
+
+        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
 
         // Duplicates can exist due table extraction colums over multiple rows.
         Set<String> processedIds = new HashSet<>();
 
         entityLoop:
-        for (Entity entity : classifiedDoc.getEntities().get(page)) {
+        for (Entity entity : entities.get(page)) {
 
             List<Comment> comments = null;
 
@@ -136,9 +140,11 @@ public class RedactionLogCreatorService {
                                 redactionLogEntry.setRedacted(false);
                                 redactionLogEntry.setStatus(Status.APPROVED);
                                 manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
                             } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
                                 manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
                                 redactionLogEntry.setStatus(Status.REQUESTED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
                             } else {
                                 redactionLogEntry.setStatus(Status.DECLINED);
                             }
@@ -149,7 +155,34 @@ public class RedactionLogCreatorService {
                             redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
                         }
                     }
+                }
 
+                if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
+                    for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
+                        if (manualForceRedact.getId().equals(entityPositionSequence.getId())) {
+                            String manualOverrideReason = null;
+                            if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
+                                entity.setRedaction(true);
+                                redactionLogEntry.setRedacted(true);
+                                redactionLogEntry.setStatus(Status.APPROVED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
+                                manualOverrideReason = entity.getRedactionReason() + ", forced by manual override";
+                                redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+                            } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
+                                manualOverrideReason = entity.getRedactionReason() + ", requested to force redact";
+                                redactionLogEntry.setStatus(Status.REQUESTED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
+                                redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+                            } else {
+                                redactionLogEntry.setStatus(Status.DECLINED);
+                            }
+
+                            entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
+                            redactionLogEntry.setReason(manualOverrideReason);
+                            redactionLogEntry.setManual(true);
+                            redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
+                        }
+                    }
                 }
 
                 if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
@@ -171,10 +204,12 @@ public class RedactionLogCreatorService {
 
                 // FIXME ids should never be null. Figure out why this happens.
                 if (redactionLogEntry.getId() != null) {
-                    classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
+                    redactionLogEntities.add(redactionLogEntry);
                 }
             }
         }
+
+        return redactionLogEntities;
     }
 
 
@@ -203,14 +238,16 @@ public class RedactionLogCreatorService {
     }
 
 
-    private void addManualEntries(Document classifiedDoc, ManualRedactions manualRedactions, int page,
+    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,  Map<String, List<Comment>> comments, int page,
                                   String ruleSetId) {
 
-        if (manualRedactions == null) {
-            return;
+        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
+
+        if (manualAdds == null) {
+            return redactionLogEntities;
         }
 
-        for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
+        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
 
             String id = manualRedactionEntry.getId();
 
@@ -224,11 +261,13 @@ public class RedactionLogCreatorService {
                 }
             }
 
-            redactionLogEntry.setComments(manualRedactions.getComments().get(id));
+            redactionLogEntry.setComments(comments.get(id));
             if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) {
-                classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
+                redactionLogEntities.add(redactionLogEntry);
             }
         }
+
+        return redactionLogEntities;
     }
 
 
@@ -277,6 +316,8 @@ public class RedactionLogCreatorService {
                 .isDictionaryEntry(entity.isDictionaryEntry())
                 .textAfter(entity.getTextAfter())
                 .textBefore(entity.getTextBefore())
+                .startOffset(entity.getStart())
+                .endOffset(entity.getEnd())
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -5,12 +5,14 @@ import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
@@ -20,8 +22,40 @@ import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @UtilityClass
+@SuppressWarnings("PMD")
 public class EntitySearchUtils {
 
+
+    public boolean sectionContainsAny(String sectionText, Set<DictionaryIncrementValue> values) {
+
+        String inputString = sectionText.toLowerCase(Locale.ROOT);
+
+        for (DictionaryIncrementValue value : values) {
+
+            String cleanValue = value.getValue().toLowerCase(Locale.ROOT).trim();
+
+            if (cleanValue.length() <= 2) {
+                continue;
+            }
+
+            int startIndex;
+            int stopIndex = 0;
+            do {
+                startIndex = inputString.indexOf(cleanValue, stopIndex);
+                stopIndex = startIndex + cleanValue.length();
+
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+                    if(value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())){
+                        return true;
+                    }
+                }
+            } while (startIndex > -1);
+        }
+        return false;
+    }
+
+
     public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
                             boolean local) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -15,6 +15,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -28,6 +29,7 @@ public class SectionsBuilderService {
         List<Paragraph> chunkBlockList = new ArrayList<>();
         List<Header> headers = new ArrayList<>();
         List<Footer> footers = new ArrayList<>();
+        List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
 
         AbstractTextContainer prev = null;
 
@@ -36,6 +38,7 @@ public class SectionsBuilderService {
         for (Page page : document.getPages()) {
             List<TextBlock> header = new ArrayList<>();
             List<TextBlock> footer = new ArrayList<>();
+            List<TextBlock> unclassifiedText = new ArrayList<>();
             for (AbstractTextContainer current : page.getTextBlocks()) {
 
                 if (current.getClassification() == null) {
@@ -54,6 +57,11 @@ public class SectionsBuilderService {
                     continue;
                 }
 
+                if (current.getClassification().equals("Other")) {
+                    unclassifiedText.add((TextBlock) current);
+                    continue;
+                }
+
                 if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification()
                         .startsWith("H ") || !document.isHeadlines()) {
                     Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
@@ -76,8 +84,16 @@ public class SectionsBuilderService {
                 chunkWords.add(current);
                 prev = current;
             }
-            headers.add(new Header(header));
-            footers.add(new Footer(footer));
+
+            if(!header.isEmpty()) {
+                headers.add(new Header(header));
+            }
+            if(!footer.isEmpty()) {
+                footers.add(new Footer(footer));
+            }
+            if(!unclassifiedText.isEmpty()) {
+                unclassifiedTexts.add(new UnclassifiedText(unclassifiedText));
+            }
         }
 
         Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
@@ -87,6 +103,7 @@ public class SectionsBuilderService {
         document.setParagraphs(chunkBlockList);
         document.setHeaders(headers);
         document.setFooters(footers);
+        document.setUnclassifiedTexts(unclassifiedTexts);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java

@@ -11,10 +11,12 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
 
 @SuppressWarnings("serial")
 @Data
 @EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor
 public class Cell extends Rectangle {
 
     private List<TextBlock> textBlocks = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -4,8 +4,12 @@ import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.when;
 import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
 
+import java.awt.Color;
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Rectangle2D;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -17,6 +21,7 @@ import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -24,7 +29,15 @@ import java.util.UUID;
 import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
+import org.apache.pdfbox.text.PDFTextStripperByArea;
+import org.apache.pdfbox.util.Matrix;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
@@ -32,6 +45,7 @@ import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
+import org.mockito.MockitoAnnotations;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.context.TestConfiguration;
@@ -40,7 +54,9 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.configuration.v1.api.model.Colors;
+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
@@ -51,19 +67,32 @@ import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.model.Comment;
 import com.iqser.red.service.redaction.v1.model.IdRemoval;
+import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.ReanalyzeResult;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.model.RenalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.model.SectionText;
 import com.iqser.red.service.redaction.v1.model.Status;
+import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.parsing.PDFAreaTextStripper;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ReanalysisSection;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(webEnvironment = RANDOM_PORT)
@@ -93,6 +122,9 @@ public class RedactionIntegrationTest {
     @Autowired
     private RedactionController redactionController;
 
+    @Autowired
+    private ObjectMapper objectMapper;
+
     @MockBean
     private RulesClient rulesClient;
 
@@ -106,6 +138,7 @@ public class RedactionIntegrationTest {
     private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
     private final Map<String, Integer> rankTypeMap = new HashMap<>();
     private final Colors colors = new Colors();
+    private final Map<String, Long> reanlysisVersions = new HashMap<>();
 
     private final static String TEST_RULESET_ID = "123";
 
@@ -324,7 +357,6 @@ public class RedactionIntegrationTest {
         recommendationTypeMap.put(PURITY, false);
         recommendationTypeMap.put(IMAGE, false);
 
-
         rankTypeMap.put(FALSE_POSITIVE, 160);
         rankTypeMap.put(PURITY, 155);
         rankTypeMap.put(PII, 150);
@@ -342,7 +374,6 @@ public class RedactionIntegrationTest {
         rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
         rankTypeMap.put(IMAGE, 30);
 
-
         colors.setDefaultColor("#acfc00");
         colors.setNotRedacted("#cccccc");
         colors.setRequestAdd("#04b093");
@@ -372,7 +403,7 @@ public class RedactionIntegrationTest {
 
         return DictionaryResponse.builder()
                 .hexColor(typeColorMap.get(type))
-                .entries(dictionary.get(type))
+                .entries(toDictionaryEntry(dictionary.get(type)))
                 .isHint(hintTypeMap.get(type))
                 .isCaseInsensitive(caseInSensitiveMap.get(type))
                 .isRecommendation(recommendationTypeMap.get(type))
@@ -381,6 +412,15 @@ public class RedactionIntegrationTest {
     }
 
 
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries){
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, false));
+        });
+        return dictionaryEntries;
+    }
+
+
     @Test
     public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
 
@@ -409,6 +449,23 @@ public class RedactionIntegrationTest {
             duplicates.entrySet().forEach(entry -> {
                 assertThat(entry.getValue().size()).isEqualTo(1);
             });
+
+
+            dictionary.get(AUTHOR).add("Drinking water");
+            when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(1L);
+
+            long rstart = System.currentTimeMillis();
+            ReanalyzeResult reanalyzeResult = redactionController.reanalyze(RenalyzeRequest.builder()
+                    .redactionLog(result.getRedactionLog())
+                    .document(IOUtils.toByteArray(new FileInputStream(path)))
+                    .manualRedactions(null)
+                    .text(result.getText())
+                    .ruleSetId(TEST_RULESET_ID)
+                    .build());
+
+            long rend = System.currentTimeMillis();
+            System.out.println("reanalysis analysis duration: " + (rend - rstart));
+
         }
 
         long end = System.currentTimeMillis();
@@ -418,7 +475,6 @@ public class RedactionIntegrationTest {
     }
 
 
-
     private List<File> getPathsRecursively(File path) {
 
         List<File> result = new ArrayList<>();
@@ -440,39 +496,206 @@ public class RedactionIntegrationTest {
     @Test
     public void redactionTest() throws IOException {
 
-//        49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
-//        182 Fludioxonil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
-//        38 A14325E - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf
-//        91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf
-//        95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf
-
-
         System.out.println("redactionTest");
         long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_13_Volume_3CP_A9396G_B-1_2018-09-06.pdf");
 
         AnalyzeRequest request = AnalyzeRequest.builder()
                 .ruleSetId(TEST_RULESET_ID)
                 .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                 .build();
 
-
         AnalyzeResult result = redactionController.analyze(request);
 
-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
+        long end = System.currentTimeMillis();
 
+        System.out.println("first analysis duration: " + (end - start));
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) {
+            fileOutputStream.write(objectMapper.writeValueAsBytes(result.getText()));
+        }
+
+        int correctFound = 0;
+        loop:
+        for (RedactionLogEntry redactionLogEntry : result.getRedactionLog().getRedactionLogEntry()) {
+            for (SectionText sectionText : result.getText().getSectionTexts()) {
+                if (redactionLogEntry.getType().equals("image")) {
+                    correctFound++;
+                    continue loop;
+                }
+                if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
+                    String value = sectionText.getText()
+                            .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
+                    if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
+                        correctFound++;
+                    } else {
+                        throw new RuntimeException("WTF");
+                    }
+                }
+            }
+        }
+        assertThat(correctFound).isEqualTo(result.getRedactionLog().getRedactionLogEntry().size());
+
+
+        dictionary.get(AUTHOR).add("properties");
+        reanlysisVersions.put("properties", 1L);
+
+        dictionary.get(AUTHOR).add("physical");
+        reanlysisVersions.put("physical", 2L);
+
+        dictionary.get(VERTEBRATE).add("s-metolachlor");
+        reanlysisVersions.put("s-metolachlor", 3L);
+
+        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(3L);
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));
+
+        start = System.currentTimeMillis();
+        ReanalyzeResult reanalyzeResult = redactionController.reanalyze(RenalyzeRequest.builder()
+                .redactionLog(result.getRedactionLog())
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .manualRedactions(null)
+                .text(result.getText())
+                .ruleSetId(TEST_RULESET_ID)
+                .build());
+
+        end = System.currentTimeMillis();
+        System.out.println("reanalysis analysis duration: " + (end - start));
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(reanalyzeResult.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
+    }
+
+
+
+    @Test
+    @Ignore
+    public void fillRecanTest() throws IOException {
+
+        System.out.println("redactionTest");
+        long start = System.currentTimeMillis();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/S5.pdf");
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .ruleSetId(TEST_RULESET_ID)
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
+
+        AnalyzeResult result = redactionController.analyze(request);
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) {
+            fileOutputStream.write(objectMapper.writeValueAsBytes(result.getText()));
+        }
+
+        int correctFound = 0;
+        loop:
+        for (RedactionLogEntry redactionLogEntry : result.getRedactionLog().getRedactionLogEntry()) {
+            for (SectionText sectionText : result.getText().getSectionTexts()) {
+                if (redactionLogEntry.getType().equals("image")) {
+                    correctFound++;
+                    continue loop;
+                }
+                if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
+                    String value = sectionText.getText()
+                            .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
+                    if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
+                        correctFound++;
+                    } else {
+                        throw new RuntimeException("WTF");
+                    }
+                }
+            }
+        }
+        assertThat(correctFound).isEqualTo(result.getRedactionLog().getRedactionLogEntry().size());
+
+        System.out.println("correctFound " + correctFound);
+
         long end = System.currentTimeMillis();
 
         System.out.println("duration: " + (end - start));
         System.out.println("numberOfPages: " + result.getNumberOfPages());
+
+        SectionArea sectionArea = result.getText().getSectionTexts().get(3).getSectionAreas().get(5);
+
+        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(IOUtils.toByteArray(pdfFileResource.getInputStream())))) {
+
+            PDPage docPage = pdDocument.getPage(0);
+
+            PDFTextStripperByArea textStripper = new PDFTextStripperByArea();
+
+            PDRectangle cropBox = docPage.getCropBox();
+            PDRectangle mediaBox = docPage.getMediaBox();
+
+
+//            if (textPositions.get(0).getRotation() == 90) {
+//                posXEnd = textPositions.get(0).getYDirAdj() + 2;
+//                posYInit = getY1();
+//                posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
+//            } else {
+//                posXEnd = textPositions.get(textPositions.size() - 1)
+//                        .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
+//                posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
+//                posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
+//                        .getYDirAdj() + 2;
+//            }
+
+
+            Rectangle2D rect = new Rectangle2D.Float(sectionArea.getTopLeft()
+                    .getY(), sectionArea.getTopLeft()
+                    .getX() , sectionArea.getHeight(), sectionArea
+                    .getWidth() + 0.001f);
+
+            textStripper.addRegion("region", rect);
+
+
+
+            textStripper.extractRegions(docPage);
+
+            String textForRegion = textStripper.getTextForRegion("region");
+
+            System.out.println(textForRegion);
+
+            // fill a rectangle
+            PDPageContentStream contents = new PDPageContentStream (pdDocument, docPage, PDPageContentStream.AppendMode.APPEND, false, false);
+            contents.setNonStrokingColor (Color.RED);
+            contents.addRect (sectionArea.getTopLeft().getX(), sectionArea.getTopLeft().getY(), sectionArea.getWidth(), sectionArea.getHeight());
+            contents.fill ();
+            contents.close ();
+            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
+                pdDocument.save(byteArrayOutputStream);
+                try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated2.pdf")) {
+                    fileOutputStream.write(byteArrayOutputStream.toByteArray());
+                }
+            }
+
+        } catch (Exception e) {
+            throw new RedactionException(e);
+        }
+
     }
 
 
+
+
+
     @Test
     public void testTableRedaction() throws IOException {
 
@@ -485,11 +708,13 @@ public class RedactionIntegrationTest {
                 .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                 .build();
 
-
         AnalyzeResult result = redactionController.analyze(request);
 
-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -504,6 +729,8 @@ public class RedactionIntegrationTest {
     @Test
     public void testManualRedaction() throws IOException {
 
+//        675eba69b0c2917de55462c817adaa05
+
         System.out.println("testManualRedaction");
         long start = System.currentTimeMillis();
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
@@ -518,8 +745,13 @@ public class RedactionIntegrationTest {
                 .text("This is a comment test")
                 .build();
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
-                .id("0836727c3508a0b2ea271da69c04cc2f")
-                .status(Status.REQUESTED)
+                .id("5b940b2cb401ed9f5be6fc24f6e77bcf")
+                .status(Status.DECLINED)
+                .build()));
+        manualRedactions.setForceRedacts(Set.of(ManualForceRedact.builder()
+                .id("675eba69b0c2917de55462c817adaa05")
+                .legalBasis("Something")
+                .status(Status.APPROVED)
                 .build()));
 
         manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
@@ -534,7 +766,7 @@ public class RedactionIntegrationTest {
         manualRedactionEntry.setReason("Manual Redaction");
         manualRedactionEntry.setPositions(List.of(new Rectangle(new Point(375.61096f, 241.282f), 7.648041f, 43.72262f, 1), new Rectangle(new Point(384.83517f, 241.282f), 7.648041f, 17.043358f, 1)));
 
-        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
+//        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
 
         AnalyzeRequest request = AnalyzeRequest.builder()
                 .ruleSetId(TEST_RULESET_ID)
@@ -542,12 +774,30 @@ public class RedactionIntegrationTest {
                 .manualRedactions(manualRedactions)
                 .build();
 
-
         AnalyzeResult result = redactionController.analyze(request);
 
-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
+        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
+                .id("5b940b2cb401ed9f5be6fc24f6e77bcf")
+                .status(Status.APPROVED)
+                .build()));
 
 
+        ReanalyzeResult reanalyzeResult = redactionController.reanalyze(RenalyzeRequest.builder()
+                .redactionLog(result.getRedactionLog())
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .manualRedactions(manualRedactions)
+                .text(result.getText())
+                .ruleSetId(TEST_RULESET_ID)
+                .build());
+
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(reanalyzeResult.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());
+
         try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
@@ -640,7 +890,6 @@ public class RedactionIntegrationTest {
                 .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                 .build();
 
-
         AnalyzeResult result = redactionController.analyze(request);
 
         result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
@@ -662,11 +911,13 @@ public class RedactionIntegrationTest {
                 .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                 .build();
 
-
         AnalyzeResult result = redactionController.analyze(request);
 
-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import com.iqser.red.service.configuration.v1.api.model.Colors;
+import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
@@ -129,12 +130,12 @@ public class EntityRedactionServiceTest {
                 .build();
 
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
+                .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H.")))
                 .build();
         when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
+                .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
 
@@ -162,12 +163,12 @@ public class EntityRedactionServiceTest {
                 .build();
 
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
+                .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H.")))
                 .build();
         when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
+                .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
         DictionaryResponse sponsorResponse = DictionaryResponse.builder()
@@ -191,11 +192,11 @@ public class EntityRedactionServiceTest {
                 " Supplement - Identity of the active substance - Reference list.pdf");
         when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                 .build();
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
         DictionaryResponse sponsorResponse = DictionaryResponse.builder()
@@ -228,15 +229,15 @@ public class EntityRedactionServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
         when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                 .build();
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
         DictionaryResponse sponsorResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))))
                 .build();
         when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
         try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
@@ -297,11 +298,11 @@ public class EntityRedactionServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
         when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                 .build();
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
         DictionaryResponse sponsorResponse = DictionaryResponse.builder()
@@ -346,7 +347,7 @@ public class EntityRedactionServiceTest {
                 .build();
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(authorResponse);
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
+                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))))
                 .build();
         when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
@@ -367,13 +368,13 @@ public class EntityRedactionServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Header Propagation.pdf");
 
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Bissig R.", "Thanei P."))
+                .entries(toDictionaryEntry(Arrays.asList("Bissig R.", "Thanei P.")))
                 .build();
 
         when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
+                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
 
@@ -392,13 +393,13 @@ public class EntityRedactionServiceTest {
         pdfFileResource = new ClassPathResource("files/Minimal Examples/Header Propagation2.pdf");
 
         dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C."))
+                .entries(toDictionaryEntry(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C.")))
                 .build();
 
         when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
+                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
 
@@ -419,13 +420,13 @@ public class EntityRedactionServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Empty Tabular Data.pdf");
 
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Aldershof S."))
+                .entries(toDictionaryEntry(Collections.singletonList("Aldershof S.")))
                 .build();
 
         when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
+                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
 
@@ -517,4 +518,12 @@ public class EntityRedactionServiceTest {
         }
     }
 
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries){
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(new DictionaryEntry(entry, 1L, false));
+        });
+        return dictionaryEntries;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl

@@ -20,145 +20,69 @@ rule "1: Redact CBI Authors"
     when
         Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
     then
-        section.redact("CBI_author", 1, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redact("CBI_author", 1, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "2: Redacted because Section contains Vertebrate"
+rule "2: Redact CBI Address"
     when
-        Section(matchesType("vertebrate"))
+        Section(matchesType("CBI_address") || matchesType("recommendation_CBI_address"))
     then
-        section.redact("CBI_address", 2, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redact("CBI_address", 2, "Address found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "3: Not Redacted because Section contains no Vertebrate"
+rule "3: Do not redact genitive CBI_author"
     when
-        Section(!matchesType("vertebrate"))
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
     then
-        section.redactNot("CBI_address", 3, "No Vertebrate found");
+        section.expandToHintAnnotationByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, "test_method");
+        section.expandToHintAnnotationByRegEx("recommendation_CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, "test_method");
     end
 
 
-rule "4: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
-    then
-        section.redactNot("CBI_address", 4, "Vertebrate and No Redaction Indicator found");
-    end
-
-
-rule "5: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
-    then
-        section.redact("CBI_address", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "6: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
+rule "4: Redact Author(s) cells in Tables with Author(s) header"
     when
         Section(hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "7: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
+rule "5: Redact Author cells in Tables with Author header"
     when
         Section(hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author", 7, "CBI_author", false, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "8: Not redacted because Vertebrate Study = N"
+rule "6: Redact and recommand Authors in Tables with Vertebrate study Y/N header"
     when
-        Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
+        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
     then
-        section.redactCell("Author(s)", 8, "CBI_author", true, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redactNot("CBI_address", 8, "Not redacted because row is not a vertebrate study");
-        section.highlightCell("Vertebrate study Y/N", 8, "hint_only");
+        section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "9: Redact if must redact entry is found"
-    when
-        Section(matchesType("must_redact"))
-    then
-        section.redact("CBI_address", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "10: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
-    then
-        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 10, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
-    end
-
-
-rule "11: Redact sponsor company"
-    when
-        Section(searchText.toLowerCase().contains("batches produced at"))
-    then
-        section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 11, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("batches produced at", "must_redact");
-    end
-
-
-rule "12: Redact determination of residues"
-    when
-        Section((
-            searchText.toLowerCase.contains("determination of residues") ||
-            searchText.toLowerCase.contains("determination of total residues")
-        ) && (
-            searchText.toLowerCase.contains("livestock") ||
-            searchText.toLowerCase.contains("live stock") ||
-            searchText.toLowerCase.contains("tissue") ||
-            searchText.toLowerCase.contains("tissues") ||
-            searchText.toLowerCase.contains("liver") ||
-            searchText.toLowerCase.contains("muscle") ||
-            searchText.toLowerCase.contains("bovine") ||
-            searchText.toLowerCase.contains("ruminant") ||
-            searchText.toLowerCase.contains("ruminants")
-        ))
-    then
-        section.redact("CBI_address", 12, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("determination of residues", "must_redact");
-        section.addHintAnnotation("determination of total residues", "must_redact");
-        section.addHintAnnotation("livestock", "must_redact");
-        section.addHintAnnotation("live stock", "must_redact");
-        section.addHintAnnotation("tissue", "must_redact");
-        section.addHintAnnotation("tissues", "must_redact");
-        section.addHintAnnotation("liver", "must_redact");
-        section.addHintAnnotation("muscle", "must_redact");
-        section.addHintAnnotation("bovine", "must_redact");
-        section.addHintAnnotation("ruminant", "must_redact");
-        section.addHintAnnotation("ruminants", "must_redact");
-    end
-
-
-rule "13: Redact if CTL/* or BL/*  was found"
+rule "7: Redact if CTL/* or BL/*  was found"
     when
         Section(searchText.contains("CTL/") || searchText.contains("BL/"))
     then
-        section.redact("CBI_address", 13, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("CTL", "must_redact");
-        section.addHintAnnotation("BL", "must_redact");
+        section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" );
+        section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" );
     end
 
 
-rule "14: Redact and add recommendation for et al. author"
+rule "8: Redact and add recommendation for et al. author"
     when
         Section(searchText.contains("et al"))
     then
-		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
 
 
-rule "15: Add recommendation for Addresses in Test Organism sections"
+rule "9: Add recommendation for Addresses in Test Organism sections"
     when
         Section(searchText.contains("Species:") && searchText.contains("Source:"))
     then
@@ -166,7 +90,7 @@ rule "15: Add recommendation for Addresses in Test Organism sections"
     end
 
 
-rule "16: Add recommendation for Addresses in Test Animals sections"
+rule "10: Add recommendation for Addresses in Test Animals sections"
     when
         Section(searchText.contains("Species") && searchText.contains("Source"))
     then
@@ -174,34 +98,35 @@ rule "16: Add recommendation for Addresses in Test Animals sections"
     end
 
 
-rule "17: Do not redact Names and Addresses if no redaction Indicator is contained"
+rule "11: Do not redact Names and Addresses if Published Information found"
     when
         Section(matchesType("published_information"))
     then
-        section.redactNot("CBI_author", 17, "Vertebrate and Published Information found");
-        section.redactNot("CBI_address", 17, "Vertebrate and Published Information found");
+        section.redactNot("CBI_author", 11, "Published Information found");
+        section.redactNot("CBI_address", 11, "Published Information found");
     end
 
+
 // --------------------------------------- PII rules -------------------------------------------------------------------
 
 
-rule "18: Redacted PII Personal Identification Information"
+rule "12: Redacted PII Personal Identification Information"
     when
         Section(matchesType("PII"))
     then
-        section.redact("PII", 18, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redact("PII", 12, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "19: Redact Emails by RegEx"
+rule "13: Redact Emails by RegEx"
     when
         Section(searchText.contains("@"))
     then
-        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 19, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "20: Redact contact information"
+rule "14: Redact contact information"
     when
        Section(text.contains("Contact point:")
        || text.contains("Phone:")
@@ -217,108 +142,120 @@ rule "20: Redact contact information"
        || text.contains("Telephone No:")
        || text.contains("Fax number:")
        || text.contains("Telephone:")
+       || text.contains("Phone No.")
        || text.contains("European contact:"))
     then
-          section.redactLineAfter("Contact point:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Phone:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel.:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Email:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("e-mail:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail address:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Alternative contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone number:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone No:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax number:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("No:", "Fax", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("Contact:", "Tel.:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("European contact:",  "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+          section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
      end
 
 
-rule "21: Redact contact information if applicant is found"
+rule "15: Redact contact information if applicant is found"
     when
         Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
     then
-        section.redactLineAfter("Contact point:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel.:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Email:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("e-mail:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail address:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Alternative contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone No:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("Contact:", "Tel.:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("European contact:",  "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "22: Redact contact information if Producer is found"
+rule "16: Redact contact information if Producer is found"
     when
         Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
     then
-        section.redactLineAfter("Contact:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "23: Redact AUTHOR(S)"
+rule "17: Redact AUTHOR(S)"
     when
         Section(searchText.contains("AUTHOR(S):"))
     then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 23, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "24: Redact PERFORMING LABORATORY"
+rule "18: Redact PERFORMING LABORATORY"
     when
         Section(searchText.contains("PERFORMING LABORATORY:"))
     then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 24, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "25: Redact On behalf of Sequani Ltd.:"
+rule "19: Redact On behalf of Sequani Ltd.:"
     when
         Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
     then
-        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 25, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
-rule "26: Redact On behalf of Syngenta Ltd.:"
+rule "20: Redact On behalf of Syngenta Ltd.:"
     when
         Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
     then
-        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 26, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // --------------------------------------- other rules -------------------------------------------------------------------
 
-rule "27: Redact Purity"
+rule "21: Purity Hint"
     when
-        Section(searchText.contains("purity"))
+        Section(searchText.toLowerCase().contains("purity"))
     then
-	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 27, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+    end
+
+
+rule "22: Redact Must Redact"
+    when
+        Section(matchesType("must_redact"))
+    then
+        section.redact("must_redact", 22, "Must Redact found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml

@@ -10,7 +10,7 @@
         <Root level="warn">
             <AppenderRef ref="CONSOLE" />
         </Root>
-        <Logger name="com.iqser.gin4" level="info" />
+        <Logger name="com.iqser" level="info" />
     </Loggers>
 
 </Configuration>