Integrate image classification

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
-import java.awt.geom.Rectangle2D;
 import java.util.List;
 
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
 
@@ -17,7 +17,7 @@ public class Page {
     @NonNull
     private List<AbstractTextContainer> textBlocks;
 
-    private List<Rectangle2D> imageBounds;
+    private List<PdfImage> images;
 
     private Rectangle bodyTextFrame;
 
@@ -31,7 +31,9 @@ public class Page {
     private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
 
+
     public boolean isRotated() {
+
         return rotation != 0;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationClient.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.multipart.MultipartFile;
+
+@FeignClient(name = "ImageClassificationResource", url = "http://localhost:8080")
+public interface ImageClassificationClient {
+
+    @PostMapping(value = "/process_full_img", consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    ImageClassificationResponse classify(@RequestBody MultipartFile file);
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationResponse.java

@@ -0,0 +1,13 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImageClassificationResponse {
+
+    private String category;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java

@@ -41,6 +41,7 @@ import org.apache.pdfbox.text.TextPosition;
 import org.apache.pdfbox.util.Matrix;
 
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 
 import lombok.Getter;
@@ -58,7 +59,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
 
     @Getter
     private int maxCharWidth;
-    
+
     @Getter
     private int minCharHeight;
 
@@ -74,7 +75,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
     private final List<Ruling> graphicsPath = new ArrayList<>();
 
     @Getter
-    private List<Rectangle2D> imageBounds = new ArrayList<>();
+    private List<PdfImage> images = new ArrayList<>();
 
     private float path_x;
     private float path_y;
@@ -222,7 +223,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                         .getWidth(), (float) imageBounds.getHeight());
 
                 if (rect.getHeight() > 2 && rect.getWidth() > 2) {
-                    this.imageBounds.add(rect);
+                    this.images.add(new PdfImage(pdfImage.getImage(), rect));
                 }
             }
         } catch (Exception e) {
@@ -358,7 +359,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
         minCharHeight = Integer.MAX_VALUE;
         maxCharHeight = 0;
         textPositionSequences.clear();
-        imageBounds = new ArrayList<>();
+        images = new ArrayList<>();
         rulings.clear();
         graphicsPath.clear();
         path_x = 0.0f;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/ParsedElements.java

@@ -1,24 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.parsing.model;
-
-import java.awt.geom.Rectangle2D;
-import java.util.List;
-
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
-
-import lombok.Builder;
-import lombok.Data;
-
-@Data
-@Builder
-public class ParsedElements {
-
-    private List<TextPositionSequence> sequences;
-    private List<Ruling> rulings;
-    private List<Rectangle2D> imageBounds;
-
-    private boolean landscape;
-    private boolean rotated;
-
-    private float minCharWidth;
-    private float maxCharWidth;
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ImageType.java

@@ -0,0 +1,5 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+public enum ImageType {
+    LOGO, FORMULA, SIGNATURE, OTHER, OCR
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.awt.geom.Rectangle2D;
+import java.awt.image.BufferedImage;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class PdfImage {
+
+    private BufferedImage image;
+    private Rectangle2D position;
+    private ImageType imageType;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImageClassificationService.java

@@ -0,0 +1,53 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+import javax.imageio.ImageIO;
+
+import org.apache.commons.fileupload.FileItem;
+import org.apache.commons.fileupload.disk.DiskFileItem;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.multipart.commons.CommonsMultipartFile;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
+import com.iqser.red.service.redaction.v1.server.client.ImageClassificationResponse;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ImageClassificationService {
+
+    private ImageClassificationClient imageClassificationClient;
+    private File repository = new File(System.getProperty("java.io.tmpdir"));
+
+
+    public void classifyImages(Document classifiedDoc) {
+
+        classifiedDoc.getPages().forEach(page -> {
+            page.getImages().forEach(image -> {
+                try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+                    ImageIO.write(image.getImage(), "png", baos);
+                    String fileName = UUID.randomUUID().toString() + ".png";
+                    FileItem fileItem = new DiskFileItem(fileName, "image/png", true, fileName, 100000000, repository);
+                    MultipartFile multipartFile = new CommonsMultipartFile(fileItem);
+                    ImageClassificationResponse response = imageClassificationClient.classify(multipartFile);
+                    image.setImageType(ImageType.valueOf(response.getCategory()));
+                    fileItem.delete();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            });
+        });
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -63,7 +63,7 @@ public class RedactionLogCreatorService {
                 classifiedDoc.getRedactionLogEntities().addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, ruleSetId));
             }
 
-            if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
+            if (!classifiedDoc.getPages().get(page - 1).getImages().isEmpty()) {
                 addImageEntries(classifiedDoc, page, ruleSetId);
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -15,7 +15,6 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
 import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
 import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
-import com.iqser.red.service.redaction.v1.server.parsing.model.ParsedElements;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
@@ -57,19 +56,10 @@ public class PdfSegmentationService {
             int rotation = pdPage.getRotation();
             boolean isRotated = rotation != 0 && rotation != 360;
 
-            ParsedElements parsedElements = ParsedElements.builder()
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
-                    .rulings(stripper.getRulings())
+                    .getMaxCharHeight());
-                    .sequences(stripper.getTextPositionSequences())
-                    .imageBounds(stripper.getImageBounds())
-                    .minCharWidth(stripper.getMinCharWidth())
-                    .maxCharWidth(stripper.getMaxCharWidth())
-                    .landscape(isLandscape)
-                    .rotated(isRotated)
-                    .build();
 
-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight());
+            Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
-
-            Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
                     .getVertical());
             page.setRotation(rotation);
 
@@ -77,11 +67,11 @@ public class PdfSegmentationService {
 
             buildPageStatistics(page);
 
-            page.setLandscape(parsedElements.isLandscape() || parsedElements.isRotated());
+            page.setLandscape(isLandscape || isRotated);
 
             page.setPageNumber(pageNumber);
             increaseDocumentStatistics(page, document);
-            page.setImageBounds(parsedElements.getImageBounds());
+            page.setImages(stripper.getImages());
             pages.add(page);
         }