Fixed endless loop on documents that contains no text

Pull request #135 : RED-1260: Enabled to add rules and manuel redaction actions for images
Merge in RED/redaction-service from RED-1260 to master * commit '55ba351362785de41090fb4252c9ee7c4c486991': RED-1260: Enabled to add rules and manuel redaction actions for images
2021-04-16 09:42:29 +02:00 · 2021-04-15 13:08:24 +02:00 · 2021-04-15 12:51:29 +02:00 · 2021-04-09 15:11:49 +02:00 · 2021-04-09 13:37:00 +02:00 · 2021-04-09 11:44:12 +02:00
29 changed files with 747 additions and 273 deletions
@@ -43,4 +43,6 @@ public class RedactionLogEntry {
    private int startOffset;
    private int endOffset;

+    private boolean isImage;
+
 }
@@ -4,11 +4,13 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;

 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;

 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -24,8 +26,8 @@ public class Document {
    private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
    private Map<Integer, List<Entity>> entities = new HashMap<>();
    private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
-    private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
-    private StringFrequencyCounter fontCounter= new StringFrequencyCounter();
+    private FloatFrequencyCounter fontSizeCounter = new FloatFrequencyCounter();
+    private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
    private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
    private boolean headlines;

@@ -35,4 +37,7 @@ public class Document {
    private long rulesVersion;

    private List<SectionText> sectionText = new ArrayList<>();
+
+    private Map<Integer, Set<Image>> images = new HashMap<>();
+
 }
@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

-import java.awt.geom.Rectangle2D;
 import java.util.List;

+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;

@@ -17,7 +17,7 @@ public class Page {
    @NonNull
    private List<AbstractTextContainer> textBlocks;

-    private List<Rectangle2D> imageBounds;
+    private List<PdfImage> images;

    private Rectangle bodyTextFrame;

@@ -31,7 +31,9 @@ public class Page {
    private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
    private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();

+
    public boolean isRotated() {
+
        return rotation != 0;
    }

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;

+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -12,9 +13,10 @@ import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
-public class Paragraph {
+public class Paragraph implements Comparable{

    private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
+    private List<PdfImage> images = new ArrayList<>();
    private String headline;


@@ -53,4 +55,11 @@ public class Paragraph {
        return textBlocks;
    }

+
+    @Override
+    public int compareTo(Object o) {
+
+        return 0;
+    }
+
 }
@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.multipart.MultipartFile;
+
+@FeignClient(name = "ImageClassificationResource", url = "${image-service.url}")
+public interface ImageClassificationClient {
+
+    @PostMapping(value = "/process_full_img", consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    ImageClassificationResponse classify(@RequestBody MultipartFile file);
+
+}
@@ -0,0 +1,13 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImageClassificationResponse {
+
+    private String category;
+}
@@ -0,0 +1,102 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.springframework.lang.NonNull;
+import org.springframework.lang.Nullable;
+import org.springframework.util.Assert;
+import org.springframework.util.FileCopyUtils;
+import org.springframework.web.multipart.MultipartFile;
+
+public class MockMultipartFile implements MultipartFile {
+
+    private final String name;
+    private final String originalFilename;
+    @Nullable
+    private final String contentType;
+    private final byte[] content;
+
+
+    public MockMultipartFile(String name, @Nullable byte[] content) {
+
+        this(name, "", (String) null, (byte[]) content);
+    }
+
+
+    public MockMultipartFile(String name, InputStream contentStream) throws IOException {
+
+        this(name, "", (String) null, (byte[]) FileCopyUtils.copyToByteArray(contentStream));
+    }
+
+
+    public MockMultipartFile(String name, @Nullable String originalFilename, @Nullable String contentType,
+                             @Nullable byte[] content) {
+
+        Assert.hasLength(name, "Name must not be empty");
+        this.name = name;
+        this.originalFilename = originalFilename != null ? originalFilename : "";
+        this.contentType = contentType;
+        this.content = content != null ? content : new byte[0];
+    }
+
+
+    public MockMultipartFile(String name, @Nullable String originalFilename, @Nullable String contentType,
+                             InputStream contentStream) throws IOException {
+
+        this(name, originalFilename, contentType, FileCopyUtils.copyToByteArray(contentStream));
+    }
+
+
+    public String getName() {
+
+        return this.name;
+    }
+
+
+    @NonNull
+    public String getOriginalFilename() {
+
+        return this.originalFilename;
+    }
+
+
+    @Nullable
+    public String getContentType() {
+
+        return this.contentType;
+    }
+
+
+    public boolean isEmpty() {
+
+        return this.content.length == 0;
+    }
+
+
+    public long getSize() {
+
+        return (long) this.content.length;
+    }
+
+
+    public byte[] getBytes() throws IOException {
+
+        return this.content;
+    }
+
+
+    public InputStream getInputStream() throws IOException {
+
+        return new ByteArrayInputStream(this.content);
+    }
+
+
+    public void transferTo(File dest) throws IOException, IllegalStateException {
+
+        FileCopyUtils.copy(this.content, dest);
+    }
+
+}
@@ -20,6 +20,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationSer
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ImageClassificationService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
@@ -53,6 +54,7 @@ public class RedactionController implements RedactionResource {
    private final DictionaryService dictionaryService;
    private final AnnotationService annotationService;
    private final ReanalyzeService reanalyzeService;
+    private final ImageClassificationService imageClassificationService;


    @Override
@@ -65,6 +67,7 @@ public class RedactionController implements RedactionResource {

            log.info("Document structure analysis successful, starting redaction analysis...");

+            imageClassificationService.classifyImages(classifiedDoc);
            entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getRuleSetId(), analyzeRequest.getManualRedactions());
            redactionLogCreatorService.createRedactionLog(classifiedDoc, pdDocument.getNumberOfPages(), analyzeRequest.getManualRedactions(), analyzeRequest
                    .getRuleSetId());
@@ -41,6 +41,7 @@ import org.apache.pdfbox.text.TextPosition;
 import org.apache.pdfbox.util.Matrix;

 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;

 import lombok.Getter;
@@ -58,7 +59,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

    @Getter
    private int maxCharWidth;
-    
+
    @Getter
    private int minCharHeight;

@@ -74,7 +75,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private final List<Ruling> graphicsPath = new ArrayList<>();

    @Getter
-    private List<Rectangle2D> imageBounds = new ArrayList<>();
+    private List<PdfImage> images = new ArrayList<>();

    private float path_x;
    private float path_y;
@@ -222,7 +223,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                        .getWidth(), (float) imageBounds.getHeight());

                if (rect.getHeight() > 2 && rect.getWidth() > 2) {
-                    this.imageBounds.add(rect);
+                    this.images.add(new PdfImage(pdfImage.getImage(), rect, pageNumber));
                }
            }
        } catch (Exception e) {
@@ -358,7 +359,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        minCharHeight = Integer.MAX_VALUE;
        maxCharHeight = 0;
        textPositionSequences.clear();
-        imageBounds = new ArrayList<>();
+        images = new ArrayList<>();
        rulings.clear();
        graphicsPath.clear();
        path_x = 0.0f;
@@ -1,24 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.parsing.model;
-
-import java.awt.geom.Rectangle2D;
-import java.util.List;
-
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
-
-import lombok.Builder;
-import lombok.Data;
-
-@Data
-@Builder
-public class ParsedElements {
-
-    private List<TextPositionSequence> sequences;
-    private List<Ruling> rulings;
-    private List<Rectangle2D> imageBounds;
-
-    private boolean landscape;
-    private boolean rotated;
-
-    private float minCharWidth;
-    private float maxCharWidth;
-}
@@ -0,0 +1,26 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.awt.geom.Rectangle2D;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Image {
+
+    private String type;
+    private Rectangle2D position;
+    private boolean redaction;
+    private String redactionReason;
+    private String legalBasis;
+    private int matchedRule;
+    private int sectionNumber;
+    private String section;
+    private int page;
+
+}
@@ -0,0 +1,5 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+public enum ImageType {
+    LOGO, FORMULA, SIGNATURE, OTHER, OCR
+}
@@ -0,0 +1,28 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.awt.geom.Rectangle2D;
+import java.awt.image.BufferedImage;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@RequiredArgsConstructor
+public class PdfImage {
+
+    @NonNull
+    private BufferedImage image;
+    @NonNull
+    private Rectangle2D position;
+    private ImageType imageType;
+    private boolean isAppendedToParagraph;
+
+    @NonNull
+    private int page;
+
+}
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;

 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;

@@ -18,6 +20,7 @@ public class ReanalysisSection {
    private List<TextBlock> textBlocks;
    private Map<String, CellValue> tabularData = new HashMap<>();
    private List<Integer> cellStarts;
+    private Set<Image> images = new HashSet<>();


    public SearchableText getSearchableText() {
@@ -51,6 +51,9 @@ public class Section {

    private SearchableText searchableText;

+    @Builder.Default
+    private Set<Image> images = new HashSet<>();
+

    public boolean rowEquals(String headerName, String value) {

@@ -75,6 +78,12 @@ public class Section {
    }


+    public boolean matchesImageType(String type) {
+
+        return images.stream().anyMatch(image -> image.getType().equals(type));
+    }
+
+
    public boolean headlineContainsWord(String word) {

        return StringUtils.containsIgnoreCase(headline, word);
@@ -109,6 +118,19 @@ public class Section {
    }


+    public void redactImage(String type, int ruleNumber, String reason, String legalBasis) {
+
+        images.forEach(image -> {
+            if (image.getType().equals(type)) {
+                image.setRedaction(true);
+                image.setMatchedRule(ruleNumber);
+                image.setRedactionReason(reason);
+                image.setLegalBasis(legalBasis);
+            }
+        });
+    }
+
+
    public void redact(String type, int ruleNumber, String reason, String legalBasis) {

        boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
@@ -125,6 +147,18 @@ public class Section {
    }


+    public void redactNotImage(String type, int ruleNumber, String reason) {
+
+        images.forEach(image -> {
+            if (image.getType().equals(type)) {
+                image.setRedaction(false);
+                image.setMatchedRule(ruleNumber);
+                image.setRedactionReason(reason);
+            }
+        });
+    }
+
+
    public void redactNot(String type, int ruleNumber, String reason) {

        boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
@@ -140,7 +174,8 @@ public class Section {
    }


-    public void expandToHintAnnotationByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group, String asType) {
+    public void expandToHintAnnotationByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group,
+                                              String asType) {

        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);

@@ -4,6 +4,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -19,22 +20,25 @@ import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
-import com.iqser.red.service.redaction.v1.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -89,7 +93,8 @@ public class EntityRedactionService {
                        .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
-                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity.getStart(), entity.getEnd()));
+                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
+                                .getStart(), entity.getEnd()));
            }
        }

@@ -120,22 +125,24 @@ public class EntityRedactionService {
                sectionNumber.incrementAndGet();
            }
            sectionSearchableTextPairs.add(processText(classifiedDoc, paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph
-                    .getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+                    .getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, paragraph
+                    .getImages()));
            sectionNumber.incrementAndGet();
        }

        for (Header header : classifiedDoc.getHeaders()) {
-            sectionSearchableTextPairs.add(processText(classifiedDoc, header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionSearchableTextPairs.add(processText(classifiedDoc, header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>()));
            sectionNumber.incrementAndGet();
        }

        for (Footer footer : classifiedDoc.getFooters()) {
-            sectionSearchableTextPairs.add(processText(classifiedDoc, footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionSearchableTextPairs.add(processText(classifiedDoc, footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>()));
            sectionNumber.incrementAndGet();
        }

        for (UnclassifiedText unclassifiedText : classifiedDoc.getUnclassifiedTexts()) {
-            sectionSearchableTextPairs.add(processText(classifiedDoc, unclassifiedText.getSearchableText(), unclassifiedText.getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionSearchableTextPairs.add(processText(classifiedDoc, unclassifiedText.getSearchableText(), unclassifiedText
+                    .getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>()));
            sectionNumber.incrementAndGet();
        }

@@ -143,6 +150,10 @@ public class EntityRedactionService {
            Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
            documentEntities.addAll(analysedRowSection.getEntities());

+            for (Image image : analysedRowSection.getImages()) {
+                classifiedDoc.getImages().computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
+            }
+
            analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {
                if (dictionary.isRecommendation(key)) {
                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
@@ -172,7 +183,8 @@ public class EntityRedactionService {
    }


-    private List<SectionSearchableTextPair> processTablePerRow(Document classifiedDoc, Table table, ManualRedactions manualRedactions,
+    private List<SectionSearchableTextPair> processTablePerRow(Document classifiedDoc, Table table,
+                                                               ManualRedactions manualRedactions,
                                                               AtomicInteger sectionNumber, Dictionary dictionary,
                                                               boolean local,
                                                               Map<Integer, Set<Entity>> hintsPerSectionNumber) {
@@ -192,7 +204,11 @@ public class EntityRedactionService {
                }

                SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
-                        .getWidth(), (float) cell.getHeight(), cell.getTextBlocks().get(0).getSequences().get(0).getPage());
+                        .getWidth(), (float) cell.getHeight(), cell.getTextBlocks()
+                        .get(0)
+                        .getSequences()
+                        .get(0)
+                        .getPage());
                sectionText.getSectionAreas().add(sectionArea);

                addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
@@ -237,7 +253,7 @@ public class EntityRedactionService {
                    .dictionary(dictionary)
                    .build(), searchableRow));

-            if(!local) {
+            if (!local) {
                sectionText.setText(searchableRow.toString());
                sectionText.setHeadline(table.getHeadline());
                sectionText.setSectionNumber(sectionNumber.intValue());
@@ -252,7 +268,8 @@ public class EntityRedactionService {
    }


-    private List<SectionSearchableTextPair> processTableAsOneText(Document classifiedDoc, Table table, ManualRedactions manualRedactions,
+    private List<SectionSearchableTextPair> processTableAsOneText(Document classifiedDoc, Table table,
+                                                                  ManualRedactions manualRedactions,
                                                                  AtomicInteger sectionNumber, Dictionary dictionary,
                                                                  boolean local,
                                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber) {
@@ -266,9 +283,13 @@ public class EntityRedactionService {
                    continue;
                }

-                if(!local) {
+                if (!local) {
                    SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
-                            .getWidth(), (float) cell.getHeight(), cell.getTextBlocks().get(0).getSequences().get(0).getPage());
+                            .getWidth(), (float) cell.getHeight(), cell.getTextBlocks()
+                            .get(0)
+                            .getSequences()
+                            .get(0)
+                            .getPage());
                    sectionText.getSectionAreas().add(sectionArea);
                }

@@ -279,7 +300,6 @@ public class EntityRedactionService {
            }
        }

-
        Set<Entity> rowEntities = findEntities(entireTableText, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
        surroundingWordsService.addSurroundingText(rowEntities, entireTableText, dictionary);

@@ -297,7 +317,7 @@ public class EntityRedactionService {
                .dictionary(dictionary)
                .build(), entireTableText));

-        if(!local) {
+        if (!local) {
            sectionText.setText(entireTableText.toString());
            sectionText.setHeadline(table.getHeadline());
            sectionText.setSectionNumber(sectionNumber.intValue());
@@ -309,12 +329,14 @@ public class EntityRedactionService {
    }


-    private SectionSearchableTextPair processText(Document classifiedDoc, SearchableText searchableText, List<TextBlock> paragraphTextBlocks,
-                                                  String headline, ManualRedactions manualRedactions,
-                                                  AtomicInteger sectionNumber, Dictionary dictionary, boolean local,
-                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber) {
+    private SectionSearchableTextPair processText(Document classifiedDoc, SearchableText searchableText,
+                                                  List<TextBlock> paragraphTextBlocks, String headline,
+                                                  ManualRedactions manualRedactions, AtomicInteger sectionNumber,
+                                                  Dictionary dictionary, boolean local,
+                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                                  List<PdfImage> images) {

-        if(!local) {
+        if (!local) {
            SectionText sectionText = new SectionText();
            for (TextBlock paragraphTextBlock : paragraphTextBlocks) {
                SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock
@@ -345,12 +367,15 @@ public class EntityRedactionService {
                .sectionNumber(sectionNumber.intValue())
                .searchableText(searchableText)
                .dictionary(dictionary)
+                .images(images.stream()
+                        .map(image -> convert(image, sectionNumber.intValue(), headline))
+                        .collect(Collectors.toSet()))
                .build(), searchableText);
    }


    public Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
-                                     Dictionary dictionary, boolean local) {
+                                    Dictionary dictionary, boolean local) {

        Set<Entity> found = new HashSet<>();
        String searchableString = searchableText.toString();
@@ -390,4 +415,18 @@ public class EntityRedactionService {
        }
    }

+
+    private Image convert(PdfImage pdfImage, int sectionNumber, String headline) {
+
+        return Image.builder()
+                .type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType()
+                        .name()
+                        .toLowerCase(Locale.ROOT))
+                .position(pdfImage.getPosition())
+                .sectionNumber(sectionNumber)
+                .section(headline)
+                .page(pdfImage.getPage())
+                .build();
+    }
+
 }
@@ -0,0 +1,62 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import javax.imageio.ImageIO;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
+import com.iqser.red.service.redaction.v1.server.client.ImageClassificationResponse;
+import com.iqser.red.service.redaction.v1.server.client.MockMultipartFile;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ImageClassificationService {
+
+    private final ImageClassificationClient imageClassificationClient;
+    private final RedactionServiceSettings settings;
+
+
+    public void classifyImages(Document classifiedDoc) {
+
+        long start = System.currentTimeMillis();
+        classifiedDoc.getPages().forEach(page -> {
+            page.getImages().forEach(image -> {
+
+                if (settings.isEnableImageClassification()) {
+                    try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+                        ImageIO.write(image.getImage(), "png", baos);
+                        ImageClassificationResponse response = imageClassificationClient.classify(new MockMultipartFile("file", "Image.png", "image/png", baos
+                                .toByteArray()));
+                        image.setImageType(ImageType.valueOf(response.getCategory()));
+
+                    } catch (IOException e) {
+                        log.error("Could not classify image", e);
+                    }
+                } else {
+                    image.setImageType(ImageType.OTHER);
+                }
+
+                if (image.getImageType().equals(ImageType.OTHER)) {
+                    page.getTextBlocks().forEach(textblock -> {
+                        if (image.getPosition()
+                                .contains(textblock.getMinX(), textblock.getMinY(), textblock.getWidth(), textblock.getHeight())) {
+                            image.setImageType(ImageType.OCR);
+                        }
+                    });
+                }
+            });
+        });
+        log.info("Image classification took: " + (System.currentTimeMillis() - start));
+    }
+
+}
@@ -39,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ReanalysisSection;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
@@ -75,10 +76,14 @@ public class ReanalyzeService {
        }

        Set<Integer> sectionsToReanaylse = new HashSet<>();
+        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
        for (RedactionLogEntry entry : renalyzeRequest.getRedactionLog().getRedactionLogEntry()) {
            if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) {
                sectionsToReanaylse.add(entry.getSectionNumber());
            }
+            if (entry.isImage() || entry.getType().equals("image")) {
+                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
+            }
        }

        for (SectionText sectionText : renalyzeRequest.getText().getSectionTexts()) {
@@ -173,10 +178,15 @@ public class ReanalyzeService {
                }
                reanalysisSection.setTextBlocks(textBlocks);
                reanalysisSection.setTabularData(tabularData);
-                reanalysisSections.add(reanalysisSection);
+
                if (sectionText.isTable()) {
                    reanalysisSection.setCellStarts(cellStarts);
                }
+                if (imageEntries.containsKey(sectionText.getSectionNumber())) {
+                    reanalysisSection.getImages().addAll(imageEntries.get(sectionText.getSectionNumber()));
+                }
+
+                reanalysisSections.add(reanalysisSection);
            }

            //--
@@ -208,14 +218,22 @@ public class ReanalyzeService {
                        .tabularData(reanalysisSection.getTabularData())
                        .searchableText(reanalysisSection.getSearchableText())
                        .dictionary(dictionary)
+                        .images(reanalysisSection.getImages())
                        .build(), reanalysisSection.getSearchableText()));
            }

            Set<Entity> entities = new HashSet<>();
+            Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
            sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
                Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair
                        .getSection());
                entities.addAll(analysedRowSection.getEntities());
+                EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+
+                for (Image image : analysedRowSection.getImages()) {
+                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
+                }
+
            });

            Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
@@ -241,6 +259,12 @@ public class ReanalyzeService {
                    newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, renalyzeRequest
                            .getManualRedactions(), page, renalyzeRequest.getRuleSetId()));
                }
+
+                if (imagesPerPage.get(page) != null) {
+                    newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, renalyzeRequest
+                            .getManualRedactions(), page, renalyzeRequest.getRuleSetId()));
+                }
+
                newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, renalyzeRequest
                        .getRuleSetId()));
            }
@@ -248,12 +272,13 @@ public class ReanalyzeService {
            Iterator<RedactionLogEntry> itty = renalyzeRequest.getRedactionLog().getRedactionLogEntry().iterator();
            while (itty.hasNext()) {
                RedactionLogEntry entry = itty.next();
-                if (sectionsToReanaylse.contains(entry.getSectionNumber()) && !entry.getType().equals("image") || entry.getSectionNumber() == 0) {
+                if (sectionsToReanaylse.contains(entry.getSectionNumber())) {
                    itty.remove();
                }
            }

            renalyzeRequest.getRedactionLog().getRedactionLogEntry().addAll(newRedactionLogEntries);
+
            renalyzeRequest.getRedactionLog().setDictionaryVersion(dictionaryIncrement.getDictionaryVersion());

            return ReanalyzeResult.builder().redactionLog(renalyzeRequest.getRedactionLog()).build();
@@ -277,4 +302,19 @@ public class ReanalyzeService {
                .collect(Collectors.toSet());
    }

+
+    public Image convert(RedactionLogEntry entry) {
+
+        Rectangle position = entry.getPositions().get(0);
+
+        return Image.builder()
+                .type(entry.getType())
+                .position(new Rectangle2D.Float(position.getTopLeft().getX(), position.getTopLeft()
+                        .getY(), position.getWidth(), position.getHeight()))
+                .sectionNumber(entry.getSectionNumber())
+                .section(entry.getSection())
+                .page(position.getPage())
+                .build();
+    }
+
 }
@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -30,6 +29,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -41,8 +41,6 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
 public class RedactionLogCreatorService {

-    private static final String IMAGE = "image";
-
    private final DictionaryService dictionaryService;


@@ -56,37 +54,110 @@ public class RedactionLogCreatorService {
            addSectionGrid(classifiedDoc, page);

            if (classifiedDoc.getEntities().get(page) != null) {
-                classifiedDoc.getRedactionLogEntities().addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, ruleSetId));
+                classifiedDoc.getRedactionLogEntities()
+                        .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, ruleSetId));
            }

            if (manualRedactionPages.contains(page)) {
-                classifiedDoc.getRedactionLogEntities().addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, ruleSetId));
+                classifiedDoc.getRedactionLogEntities()
+                        .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, ruleSetId));
            }

-            if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
-                addImageEntries(classifiedDoc, page, ruleSetId);
+            if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) {
+                classifiedDoc.getRedactionLogEntities()
+                        .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, ruleSetId));
            }
        }
    }


-    private void addImageEntries(Document classifiedDoc, int pageNumber, String ruleSetId) {
+    public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, ManualRedactions manualRedactions,
+                                                   int pageNumber, String ruleSetId) {
+
+        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
+
+        for (Image image : images.get(pageNumber)) {
+
+            String id = IdBuilder.buildId(image.getPosition(), pageNumber);

-        for (Rectangle2D imageBounds : classifiedDoc.getPages().get(pageNumber - 1).getImageBounds()) {
            RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
-                    .id(IdBuilder.buildId(imageBounds, pageNumber))
-                    .color(getColor(IMAGE, ruleSetId))
-                    .type(IMAGE)
-                    .redacted(false)
-                    .isHint(true)
+                    .id(id)
+                    .color(getColorForImage(image, ruleSetId, false))
+                    .isImage(true)
+                    .type(image.getType())
+                    .redacted(image.isRedaction())
+                    .reason(image.getRedactionReason())
+                    .legalBasis(image.getLegalBasis())
+                    .matchedRule(image.getMatchedRule())
+                    .isHint(dictionaryService.isHint(image.getType(), ruleSetId))
                    .manual(false)
                    .isDictionaryEntry(false)
                    .isRecommendation(false)
-                    .positions(List.of(new Rectangle(new Point((float) imageBounds.getX(), (float) imageBounds.getY()), (float) imageBounds
-                            .getWidth(), (float) imageBounds.getHeight(), pageNumber)))
+                    .positions(List.of(new Rectangle(new Point((float) image.getPosition()
+                            .getX(), (float) image.getPosition().getY()), (float) image.getPosition()
+                            .getWidth(), (float) image.getPosition().getHeight(), pageNumber)))
+                    .sectionNumber(image.getSectionNumber())
+                    .section(image.getSection())
                    .build();
-            classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
+
+            if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
+                for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
+                    if (manualRemoval.getId().equals(id)) {
+                        String manualOverrideReason = null;
+                        if (manualRemoval.getStatus().equals(Status.APPROVED)) {
+                            image.setRedaction(false);
+                            redactionLogEntry.setRedacted(false);
+                            redactionLogEntry.setStatus(Status.APPROVED);
+                            manualOverrideReason = image.getRedactionReason() + ", removed by manual override";
+                            redactionLogEntry.setColor(getColorForImage(image, ruleSetId, false));
+                        } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
+                            manualOverrideReason = image.getRedactionReason() + ", requested to remove";
+                            redactionLogEntry.setStatus(Status.REQUESTED);
+                            redactionLogEntry.setColor(getColorForImage(image, ruleSetId, true));
+                        } else {
+                            redactionLogEntry.setStatus(Status.DECLINED);
+                        }
+
+                        image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
+                        redactionLogEntry.setReason(manualOverrideReason);
+                        redactionLogEntry.setManual(true);
+                        redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
+                    }
+                }
+            }
+
+            if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
+                for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
+                    if (manualForceRedact.getId().equals(id)) {
+                        String manualOverrideReason = null;
+                        if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
+                            image.setRedaction(true);
+                            redactionLogEntry.setRedacted(true);
+                            redactionLogEntry.setStatus(Status.APPROVED);
+                            redactionLogEntry.setColor(getColorForImage(image, ruleSetId, false));
+                            manualOverrideReason = image.getRedactionReason() + ", forced by manual override";
+                            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+                        } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
+                            manualOverrideReason = image.getRedactionReason() + ", requested to force redact";
+                            redactionLogEntry.setStatus(Status.REQUESTED);
+                            redactionLogEntry.setColor(getColorForImage(image, ruleSetId, true));
+                            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+                        } else {
+                            redactionLogEntry.setStatus(Status.DECLINED);
+                        }
+
+                        image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
+                        redactionLogEntry.setReason(manualOverrideReason);
+                        redactionLogEntry.setManual(true);
+                        redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
+                    }
+                }
+            }
+
+            redactionLogEntities.add(redactionLogEntry);
        }
+
+        return redactionLogEntities;
    }


@@ -107,7 +178,8 @@ public class RedactionLogCreatorService {
    }


-    public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions, int page, String ruleSetId) {
+    public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions,
+                                              int page, String ruleSetId) {

        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

@@ -238,8 +310,9 @@ public class RedactionLogCreatorService {
    }


-    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,  Map<String, List<Comment>> comments, int page,
-                                  String ruleSetId) {
+    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
+                                                       Map<String, List<Comment>> comments, int page,
+                                                       String ruleSetId) {

        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

@@ -351,6 +424,18 @@ public class RedactionLogCreatorService {
    }


+    private float[] getColorForImage(Image image, String ruleSetId, boolean requestedToRemove) {
+
+        if (requestedToRemove) {
+            return dictionaryService.getRequestRemoveColor(ruleSetId);
+        }
+        if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), ruleSetId)) {
+            return dictionaryService.getNotRedactedColor(ruleSetId);
+        }
+        return dictionaryService.getColor(image.getType(), ruleSetId);
+    }
+
+
    private boolean isHint(Entity entity, String ruleSetId) {

        return dictionaryService.isHint(entity.getType(), ruleSetId);
@@ -15,7 +15,6 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
 import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
 import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
-import com.iqser.red.service.redaction.v1.server.parsing.model.ParsedElements;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
@@ -57,19 +56,10 @@ public class PdfSegmentationService {
            int rotation = pdPage.getRotation();
            boolean isRotated = rotation != 0 && rotation != 360;

-            ParsedElements parsedElements = ParsedElements.builder()
-                    .rulings(stripper.getRulings())
-                    .sequences(stripper.getTextPositionSequences())
-                    .imageBounds(stripper.getImageBounds())
-                    .minCharWidth(stripper.getMinCharWidth())
-                    .maxCharWidth(stripper.getMaxCharWidth())
-                    .landscape(isLandscape)
-                    .rotated(isRotated)
-                    .build();
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
+                    .getMaxCharHeight());

-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight());
-
-            Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
+            Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
                    .getVertical());
            page.setRotation(rotation);

@@ -77,19 +67,20 @@ public class PdfSegmentationService {

            buildPageStatistics(page);

-            page.setLandscape(parsedElements.isLandscape() || parsedElements.isRotated());
+            page.setLandscape(isLandscape || isRotated);

            page.setPageNumber(pageNumber);
            increaseDocumentStatistics(page, document);
-            page.setImageBounds(parsedElements.getImageBounds());
+
+            page.setImages(stripper.getImages());
            pages.add(page);
        }

        document.setPages(pages);

        classificationService.classifyDocument(document);
-
        sectionsBuilderService.buildSections(document);
+        sectionsBuilderService.addImagesToSections(document);

        return document;
    }
@@ -2,8 +2,12 @@ package com.iqser.red.service.redaction.v1.server.segmentation;

 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.stream.Collectors;

 import org.apache.commons.collections4.CollectionUtils;
@@ -16,6 +20,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -85,13 +90,13 @@ public class SectionsBuilderService {
                prev = current;
            }

-            if(!header.isEmpty()) {
+            if (!header.isEmpty()) {
                headers.add(new Header(header));
            }
-            if(!footer.isEmpty()) {
+            if (!footer.isEmpty()) {
                footers.add(new Footer(footer));
            }
-            if(!unclassifiedText.isEmpty()) {
+            if (!unclassifiedText.isEmpty()) {
                unclassifiedTexts.add(new UnclassifiedText(unclassifiedText));
            }
        }
@@ -107,6 +112,59 @@ public class SectionsBuilderService {
    }


+    public void addImagesToSections(Document document) {
+
+        Map<Integer, SortedSet<Paragraph>> paragraphMap = new HashMap<>();
+        for (Paragraph paragraph : document.getParagraphs()) {
+            for (AbstractTextContainer container : paragraph.getPageBlocks()) {
+                paragraphMap.computeIfAbsent(container.getPage(), x -> new TreeSet<>()).add(paragraph);
+            }
+        }
+
+        if (paragraphMap.isEmpty()) {
+            Paragraph paragraph = new Paragraph();
+            document.getParagraphs().add(paragraph);
+            paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
+        }
+
+        for (Page page : document.getPages()) {
+            for (PdfImage image : page.getImages()) {
+                SortedSet<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
+                if (paragraphsOnPage == null) {
+                    int i = page.getPageNumber();
+                    while (paragraphsOnPage == null) {
+                        paragraphsOnPage = paragraphMap.get(i);
+                        i--;
+                    }
+                }
+
+                Float perviousEnd = 0f;
+                for (Paragraph paragraph : paragraphsOnPage) {
+                    Float currentEnd = 0f;
+                    for (AbstractTextContainer abs : paragraph.getPageBlocks()) {
+                        if (abs.getPage() != page.getPageNumber()) {
+                            continue;
+                        }
+                        if (abs.getMaxY() > currentEnd) {
+                            currentEnd = abs.getMaxY();
+                        }
+                    }
+
+                    if (image.getPosition().getY() >= perviousEnd && image.getPosition().getY() <= currentEnd) {
+                        paragraph.getImages().add(image);
+                        image.setAppendedToParagraph(true);
+                    }
+                    perviousEnd = currentEnd;
+                }
+                if (!image.isAppendedToParagraph()) {
+                    paragraphsOnPage.first().getImages().add(image);
+                    image.setAppendedToParagraph(true);
+                }
+            }
+        }
+    }
+
+
    private void mergeTableMetadata(Table currentTable, Table previousTable) {

        // Distribute header information for subsequent tables
@@ -12,4 +12,6 @@ public class RedactionServiceSettings {

    private int surroundingWordsOffsetWindow = 100;

+    private boolean enableImageClassification = true;
+
 }
@@ -2,6 +2,7 @@ info:
  description: Redaction Service Server V1

 configuration-service.url: "http://configuration-service-v1:8080"
+image-service.url: "http://image-service-v1:8080"

 server:
  port: 8080
@@ -4,12 +4,8 @@ import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.when;
 import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;

-import java.awt.Color;
-import java.awt.geom.AffineTransform;
-import java.awt.geom.Rectangle2D;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -21,7 +17,6 @@ import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -29,15 +24,7 @@ import java.util.UUID;
 import java.util.stream.Collectors;

 import org.apache.commons.io.IOUtils;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.PDPageContentStream;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
-import org.apache.pdfbox.text.PDFTextStripperByArea;
-import org.apache.pdfbox.util.Matrix;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
@@ -45,7 +32,6 @@ import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
-import org.mockito.MockitoAnnotations;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.context.TestConfiguration;
@@ -77,22 +63,14 @@ import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.model.RenalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.model.SectionText;
 import com.iqser.red.service.redaction.v1.model.Status;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.parsing.PDFAreaTextStripper;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
-import com.iqser.red.service.redaction.v1.server.redaction.model.ReanalysisSection;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;

@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = RANDOM_PORT)
@@ -111,6 +89,10 @@ public class RedactionIntegrationTest {
    private static final String TEST_METHOD = "test_method";
    private static final String PURITY = "purity";
    private static final String IMAGE = "image";
+    private static final String LOGO = "logo";
+    private static final String SIGNATURE = "signature";
+    private static final String FORMULA = "formula";
+    private static final String OCR = "ocr";

    private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
    private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
@@ -131,6 +113,9 @@ public class RedactionIntegrationTest {
    @MockBean
    private DictionaryClient dictionaryClient;

+    @MockBean
+    private ImageClassificationClient imageClassificationClient;
+
    private final Map<String, List<String>> dictionary = new HashMap<>();
    private final Map<String, String> typeColorMap = new HashMap<>();
    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
@@ -192,6 +177,10 @@ public class RedactionIntegrationTest {
        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
        when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY));
        when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(IMAGE));
+        when(dictionaryClient.getDictionaryForType(OCR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(OCR));
+        when(dictionaryClient.getDictionaryForType(LOGO, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(LOGO));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(SIGNATURE));
+        when(dictionaryClient.getDictionaryForType(FORMULA, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FORMULA));
        when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
    }

@@ -274,7 +263,27 @@ public class RedactionIntegrationTest {
                        .map(this::cleanDictionaryEntry)
                        .collect(Collectors.toSet()));
        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/image.txt")
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
                        .stream()
                        .map(this::cleanDictionaryEntry)
                        .collect(Collectors.toSet()));
@@ -305,6 +314,10 @@ public class RedactionIntegrationTest {
        typeColorMap.put(FALSE_POSITIVE, "#ffffff");
        typeColorMap.put(PURITY, "#ffe187");
        typeColorMap.put(IMAGE, "#fcc5fb");
+        typeColorMap.put(OCR, "#fcc5fb");
+        typeColorMap.put(LOGO, "#ffe187");
+        typeColorMap.put(FORMULA, "#ffe187");
+        typeColorMap.put(SIGNATURE, "#ffe187");

        hintTypeMap.put(VERTEBRATE, true);
        hintTypeMap.put(ADDRESS, false);
@@ -322,6 +335,10 @@ public class RedactionIntegrationTest {
        hintTypeMap.put(FALSE_POSITIVE, true);
        hintTypeMap.put(PURITY, false);
        hintTypeMap.put(IMAGE, true);
+        hintTypeMap.put(OCR, true);
+        hintTypeMap.put(FORMULA, false);
+        hintTypeMap.put(LOGO, false);
+        hintTypeMap.put(SIGNATURE, false);

        caseInSensitiveMap.put(VERTEBRATE, true);
        caseInSensitiveMap.put(ADDRESS, false);
@@ -339,6 +356,10 @@ public class RedactionIntegrationTest {
        caseInSensitiveMap.put(FALSE_POSITIVE, false);
        caseInSensitiveMap.put(PURITY, false);
        caseInSensitiveMap.put(IMAGE, true);
+        caseInSensitiveMap.put(OCR, true);
+        caseInSensitiveMap.put(SIGNATURE, true);
+        caseInSensitiveMap.put(LOGO, true);
+        caseInSensitiveMap.put(FORMULA, true);

        recommendationTypeMap.put(VERTEBRATE, false);
        recommendationTypeMap.put(ADDRESS, false);
@@ -356,6 +377,10 @@ public class RedactionIntegrationTest {
        recommendationTypeMap.put(FALSE_POSITIVE, false);
        recommendationTypeMap.put(PURITY, false);
        recommendationTypeMap.put(IMAGE, false);
+        recommendationTypeMap.put(OCR, false);
+        recommendationTypeMap.put(FORMULA, false);
+        recommendationTypeMap.put(SIGNATURE, false);
+        recommendationTypeMap.put(LOGO, false);

        rankTypeMap.put(FALSE_POSITIVE, 160);
        rankTypeMap.put(PURITY, 155);
@@ -373,6 +398,10 @@ public class RedactionIntegrationTest {
        rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
        rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
        rankTypeMap.put(IMAGE, 30);
+        rankTypeMap.put(OCR, 29);
+        rankTypeMap.put(LOGO, 28);
+        rankTypeMap.put(SIGNATURE, 27);
+        rankTypeMap.put(FORMULA, 26);

        colors.setDefaultColor("#acfc00");
        colors.setNotRedacted("#cccccc");
@@ -412,7 +441,8 @@ public class RedactionIntegrationTest {
    }


-    private List<DictionaryEntry> toDictionaryEntry(List<String> entries){
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
        entries.forEach(entry -> {
            dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, false));
@@ -450,7 +480,6 @@ public class RedactionIntegrationTest {
                assertThat(entry.getValue().size()).isEqualTo(1);
            });

-
            dictionary.get(AUTHOR).add("Drinking water");
            when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(1L);

@@ -498,7 +527,7 @@ public class RedactionIntegrationTest {

        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_13_Volume_3CP_A9396G_B-1_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf");

        AnalyzeRequest request = AnalyzeRequest.builder()
                .ruleSetId(TEST_RULESET_ID)
@@ -507,6 +536,12 @@ public class RedactionIntegrationTest {

        AnalyzeResult result = redactionController.analyze(request);

+        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
+            if (entry.isImage()) {
+                System.out.println("---->" + entry.getType());
+            }
+        });
+
        long end = System.currentTimeMillis();

        System.out.println("first analysis duration: " + (end - start));
@@ -519,7 +554,7 @@ public class RedactionIntegrationTest {
        loop:
        for (RedactionLogEntry redactionLogEntry : result.getRedactionLog().getRedactionLogEntry()) {
            for (SectionText sectionText : result.getText().getSectionTexts()) {
-                if (redactionLogEntry.getType().equals("image")) {
+                if (redactionLogEntry.isImage()) {
                    correctFound++;
                    continue loop;
                }
@@ -536,7 +571,6 @@ public class RedactionIntegrationTest {
        }
        assertThat(correctFound).isEqualTo(result.getRedactionLog().getRedactionLogEntry().size());

-
        dictionary.get(AUTHOR).add("properties");
        reanlysisVersions.put("properties", 1L);

@@ -554,7 +588,6 @@ public class RedactionIntegrationTest {
        ReanalyzeResult reanalyzeResult = redactionController.reanalyze(RenalyzeRequest.builder()
                .redactionLog(result.getRedactionLog())
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
-                .manualRedactions(null)
                .text(result.getText())
                .ruleSetId(TEST_RULESET_ID)
                .build());
@@ -575,127 +608,6 @@ public class RedactionIntegrationTest {
    }


-
-    @Test
-    @Ignore
-    public void fillRecanTest() throws IOException {
-
-        System.out.println("redactionTest");
-        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/S5.pdf");
-
-        AnalyzeRequest request = AnalyzeRequest.builder()
-                .ruleSetId(TEST_RULESET_ID)
-                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
-                .build();
-
-        AnalyzeResult result = redactionController.analyze(request);
-
-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
-                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
-                .redactionLog(result.getRedactionLog())
-                .sectionGrid(result.getSectionGrid())
-                .build());
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
-            fileOutputStream.write(annotateResponse.getDocument());
-        }
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) {
-            fileOutputStream.write(objectMapper.writeValueAsBytes(result.getText()));
-        }
-
-        int correctFound = 0;
-        loop:
-        for (RedactionLogEntry redactionLogEntry : result.getRedactionLog().getRedactionLogEntry()) {
-            for (SectionText sectionText : result.getText().getSectionTexts()) {
-                if (redactionLogEntry.getType().equals("image")) {
-                    correctFound++;
-                    continue loop;
-                }
-                if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
-                    String value = sectionText.getText()
-                            .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
-                    if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
-                        correctFound++;
-                    } else {
-                        throw new RuntimeException("WTF");
-                    }
-                }
-            }
-        }
-        assertThat(correctFound).isEqualTo(result.getRedactionLog().getRedactionLogEntry().size());
-
-        System.out.println("correctFound " + correctFound);
-
-        long end = System.currentTimeMillis();
-
-        System.out.println("duration: " + (end - start));
-        System.out.println("numberOfPages: " + result.getNumberOfPages());
-
-        SectionArea sectionArea = result.getText().getSectionTexts().get(3).getSectionAreas().get(5);
-
-        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(IOUtils.toByteArray(pdfFileResource.getInputStream())))) {
-
-            PDPage docPage = pdDocument.getPage(0);
-
-            PDFTextStripperByArea textStripper = new PDFTextStripperByArea();
-
-            PDRectangle cropBox = docPage.getCropBox();
-            PDRectangle mediaBox = docPage.getMediaBox();
-
-
-//            if (textPositions.get(0).getRotation() == 90) {
-//                posXEnd = textPositions.get(0).getYDirAdj() + 2;
-//                posYInit = getY1();
-//                posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
-//            } else {
-//                posXEnd = textPositions.get(textPositions.size() - 1)
-//                        .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
-//                posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-//                posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
-//                        .getYDirAdj() + 2;
-//            }
-
-
-            Rectangle2D rect = new Rectangle2D.Float(sectionArea.getTopLeft()
-                    .getY(), sectionArea.getTopLeft()
-                    .getX() , sectionArea.getHeight(), sectionArea
-                    .getWidth() + 0.001f);
-
-            textStripper.addRegion("region", rect);
-
-
-
-            textStripper.extractRegions(docPage);
-
-            String textForRegion = textStripper.getTextForRegion("region");
-
-            System.out.println(textForRegion);
-
-            // fill a rectangle
-            PDPageContentStream contents = new PDPageContentStream (pdDocument, docPage, PDPageContentStream.AppendMode.APPEND, false, false);
-            contents.setNonStrokingColor (Color.RED);
-            contents.addRect (sectionArea.getTopLeft().getX(), sectionArea.getTopLeft().getY(), sectionArea.getWidth(), sectionArea.getHeight());
-            contents.fill ();
-            contents.close ();
-            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
-                pdDocument.save(byteArrayOutputStream);
-                try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated2.pdf")) {
-                    fileOutputStream.write(byteArrayOutputStream.toByteArray());
-                }
-            }
-
-        } catch (Exception e) {
-            throw new RedactionException(e);
-        }
-
-    }
-
-
-
-
-
    @Test
    public void testTableRedaction() throws IOException {

@@ -782,7 +694,6 @@ public class RedactionIntegrationTest {
                .status(Status.APPROVED)
                .build()));

-
        ReanalyzeResult reanalyzeResult = redactionController.reanalyze(RenalyzeRequest.builder()
                .redactionLog(result.getRedactionLog())
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -791,7 +702,6 @@ public class RedactionIntegrationTest {
                .ruleSetId(TEST_RULESET_ID)
                .build());

-
        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .redactionLog(reanalyzeResult.getRedactionLog())
@@ -2,12 +2,17 @@ package com.iqser.red.service.redaction.v1.server.segmentation;

 import static org.assertj.core.api.Assertions.assertThat;

+import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;

+import javax.imageio.ImageIO;
+
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.runtime.KieContainer;
@@ -18,7 +23,10 @@ import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;

 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
@@ -45,24 +53,25 @@ public class PdfSegmentationServiceTest {


    @Test
-    public void testPDFSegmentationWithComplexTable() throws IOException {
+    @Ignore
+    public void testExtractImages() throws IOException {

-        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf");

        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
            Document document = pdfSegmentationService.parseDocument(pdDocument);
-            assertThat(document.getParagraphs()
-                    .stream()
-                    .flatMap(paragraph -> paragraph.getTables().stream())
-                    .collect(Collectors.toList())).isNotEmpty();
-            Table table = document.getParagraphs()
-                    .stream()
-                    .flatMap(paragraph -> paragraph.getTables().stream())
-                    .collect(Collectors.toList())
-                    .get(0);
-            assertThat(table.getColCount()).isEqualTo(6);
-            assertThat(table.getRowCount()).isEqualTo(13);
-            assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
+            int i = 0;
+            for (Page page : document.getPages()) {
+                for (PdfImage image : page.getImages()) {
+                    try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+                        ImageIO.write(image.getImage(), "png", baos);
+                        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Image " + i + ".png")) {
+                            fileOutputStream.write(baos.toByteArray());
+                        }
+                    }
+                    i++;
+                }
+            }
        }
    }

@@ -97,12 +106,12 @@ public class PdfSegmentationServiceTest {
                    .stream()
                    .map(Collections::singletonList)
                    .collect(Collectors.toList());
-            assertThat(secondTable.getRows().stream()
+            assertThat(secondTable.getRows()
+                    .stream()
                    .allMatch(row -> row.stream()
                            .map(Cell::getHeaderCells)
                            .collect(Collectors.toList())
-                            .equals(firstTableHeaderCells)))
-                    .isTrue();
+                            .equals(firstTableHeaderCells))).isTrue();
        }
    }

@@ -137,12 +146,12 @@ public class PdfSegmentationServiceTest {
                    .stream()
                    .map(Cell::getHeaderCells)
                    .collect(Collectors.toList());
-            assertThat(secondTable.getRows().stream()
+            assertThat(secondTable.getRows()
+                    .stream()
                    .allMatch(row -> row.stream()
                            .map(Cell::getHeaderCells)
                            .collect(Collectors.toList())
-                            .equals(firstTableHeaderCells)))
-                    .isTrue();
+                            .equals(firstTableHeaderCells))).isTrue();
        }
    }

@@ -177,12 +186,12 @@ public class PdfSegmentationServiceTest {
                    .stream()
                    .map(Collections::singletonList)
                    .collect(Collectors.toList());
-            assertThat(secondTable.getRows().stream()
+            assertThat(secondTable.getRows()
+                    .stream()
                    .allMatch(row -> row.stream()
                            .map(Cell::getHeaderCells)
                            .collect(Collectors.toList())
-                            .equals(firstTableHeaderCells)))
-                    .isTrue();
+                            .equals(firstTableHeaderCells))).isTrue();
        }
    }

@@ -1,4 +1,5 @@
 configuration-service.url: "http://configuration-service-v1:8080"
+image-service.url: "http://image-service-v1:8080"

 ribbon:
  ConnectTimeout: 600000
@@ -12,3 +13,6 @@ processing.kafkastreams: false

 platform.multi-tenancy:
  enabled: false
+
+redaction-service:
+  enable-image-classification: false
@@ -258,4 +258,28 @@ rule "22: Redact Must Redact"
        Section(matchesType("must_redact"))
    then
        section.redact("must_redact", 22, "Must Redact found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "23: Redact signatures"
+    when
+        Section(matchesImageType("signature"))
+    then
+        section.redactImage("signature", 23, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+
+rule "24: Redact formula"
+    when
+        Section(matchesImageType("formula"))
+    then
+        section.redactImage("formula", 24, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+
+rule "25: Redact Logos"
+    when
+        Section(matchesImageType("logo"))
+    then
+        section.redactImage("logo", 25, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
    end
@@ -304,4 +304,28 @@ rule "25: Redact Purity"
        Section(searchText.contains("purity"))
    then
 	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
+    end
+
+
+rule "26: Redact signatures"
+    when
+        Section(matchesImageType("signature"))
+    then
+        section.redactImage("signature", 26, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+
+rule "27: Redact formula"
+    when
+        Section(matchesImageType("formula"))
+    then
+        section.redactImage("formula", 27, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+
+rule "28: Redact Logos"
+    when
+        Section(matchesImageType("logo"))
+    then
+        section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
    end
Author	SHA1	Message	Date
Dominique Eifländer	c157a80630	Fixed endless loop on documents that contains no text	2021-04-16 09:42:29 +02:00
Dominique Eiflaender	a7aa3a723a	Pull request #135 : RED-1260: Enabled to add rules and manuel redaction actions for images Merge in RED/redaction-service from RED-1260 to master * commit '55ba351362785de41090fb4252c9ee7c4c486991': RED-1260: Enabled to add rules and manuel redaction actions for images	2021-04-15 13:08:24 +02:00
Dominique Eifländer	55ba351362	RED-1260: Enabled to add rules and manuel redaction actions for images	2021-04-15 12:51:29 +02:00
Dominique Eiflaender	9696a421fc	Pull request #134 : RED-1260 Merge in RED/redaction-service from RED-1260 to master * commit 'ae28555bf4c740d9872e26fb27615fbf7402f002': RED-1260: First steps for image classification Integrate image classification	2021-04-09 15:11:49 +02:00
Dominique Eifländer	ae28555bf4	RED-1260: First steps for image classification	2021-04-09 13:37:00 +02:00
Dominique Eifländer	2558b3cab8	Integrate image classification	2021-04-09 11:44:12 +02:00
Dominique Eiflaender	2c92f03f2a	Pull request #133 : Do not remove images on reanalysis Merge in RED/redaction-service from DoNotRemoveImages2 to master * commit 'fc70f972da887370f011ad1998e209ba676d90db': Do not remove images on reanalysis	2021-04-08 16:02:05 +02:00
Dominique Eifländer	fc70f972da	Do not remove images on reanalysis	2021-04-08 15:47:11 +02:00
Dominique Eiflaender	0e8c28f8fd	Pull request #132 : Do not remove images at reanalysis Merge in RED/redaction-service from ImageReanalysisFix to master * commit 'e7c24487c755ab84c0f1efe5a0c6e9d4794d4c0f': Do not remove images at reanalysis	2021-04-08 15:37:13 +02:00