From 2558b3cab8fd7991059be4b6645aeae50c366042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Fri, 9 Apr 2021 11:44:12 +0200 Subject: [PATCH] Integrate image classification --- .../v1/server/classification/model/Page.java | 6 ++- .../client/ImageClassificationClient.java | 15 ++++++ .../client/ImageClassificationResponse.java | 13 +++++ .../server/parsing/PDFLinesTextStripper.java | 9 ++-- .../server/parsing/model/ParsedElements.java | 24 --------- .../v1/server/redaction/model/ImageType.java | 5 ++ .../v1/server/redaction/model/PdfImage.java | 19 +++++++ .../service/ImageClassificationService.java | 53 +++++++++++++++++++ .../service/RedactionLogCreatorService.java | 2 +- .../segmentation/PdfSegmentationService.java | 20 ++----- 10 files changed, 120 insertions(+), 46 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationClient.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationResponse.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/ParsedElements.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ImageType.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImageClassificationService.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java index 89c81de5..af07f19f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java @@ -1,8 +1,8 @@ package com.iqser.red.service.redaction.v1.server.classification.model; -import java.awt.geom.Rectangle2D; import java.util.List; +import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle; @@ -17,7 +17,7 @@ public class Page { @NonNull private List textBlocks; - private List imageBounds; + private List images; private Rectangle bodyTextFrame; @@ -31,7 +31,9 @@ public class Page { private StringFrequencyCounter fontCounter = new StringFrequencyCounter(); private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter(); + public boolean isRotated() { + return rotation != 0; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationClient.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationClient.java new file mode 100644 index 00000000..a7dbc0e0 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationClient.java @@ -0,0 +1,15 @@ +package com.iqser.red.service.redaction.v1.server.client; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.multipart.MultipartFile; + +@FeignClient(name = "ImageClassificationResource", url = "http://localhost:8080") +public interface ImageClassificationClient { + + @PostMapping(value = "/process_full_img", consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + ImageClassificationResponse classify(@RequestBody MultipartFile file); + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationResponse.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationResponse.java new file mode 100644 index 00000000..81ae0643 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/ImageClassificationResponse.java @@ -0,0 +1,13 @@ +package com.iqser.red.service.redaction.v1.server.client; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ImageClassificationResponse { + + private String category; +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java index 5a4265bd..9e2c3be0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java @@ -41,6 +41,7 @@ import org.apache.pdfbox.text.TextPosition; import org.apache.pdfbox.util.Matrix; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; import lombok.Getter; @@ -58,7 +59,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { @Getter private int maxCharWidth; - + @Getter private int minCharHeight; @@ -74,7 +75,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { private final List graphicsPath = new ArrayList<>(); @Getter - private List imageBounds = new ArrayList<>(); + private List images = new ArrayList<>(); private float path_x; private float path_y; @@ -222,7 +223,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { .getWidth(), (float) imageBounds.getHeight()); if (rect.getHeight() > 2 && rect.getWidth() > 2) { - this.imageBounds.add(rect); + this.images.add(new PdfImage(pdfImage.getImage(), rect)); } } } catch (Exception e) { @@ -358,7 +359,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { minCharHeight = Integer.MAX_VALUE; maxCharHeight = 0; textPositionSequences.clear(); - imageBounds = new ArrayList<>(); + images = new ArrayList<>(); rulings.clear(); graphicsPath.clear(); path_x = 0.0f; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/ParsedElements.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/ParsedElements.java deleted file mode 100644 index 479ac2d5..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/ParsedElements.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.parsing.model; - -import java.awt.geom.Rectangle2D; -import java.util.List; - -import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class ParsedElements { - - private List sequences; - private List rulings; - private List imageBounds; - - private boolean landscape; - private boolean rotated; - - private float minCharWidth; - private float maxCharWidth; -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ImageType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ImageType.java new file mode 100644 index 00000000..a2a1843f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ImageType.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +public enum ImageType { + LOGO, FORMULA, SIGNATURE, OTHER, OCR +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java new file mode 100644 index 00000000..f345d3c9 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import java.awt.geom.Rectangle2D; +import java.awt.image.BufferedImage; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class PdfImage { + + private BufferedImage image; + private Rectangle2D position; + private ImageType imageType; + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImageClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImageClassificationService.java new file mode 100644 index 00000000..c36281c7 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImageClassificationService.java @@ -0,0 +1,53 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +import javax.imageio.ImageIO; + +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.disk.DiskFileItem; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.commons.CommonsMultipartFile; + +import com.iqser.red.service.redaction.v1.server.classification.model.Document; +import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient; +import com.iqser.red.service.redaction.v1.server.client.ImageClassificationResponse; +import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageClassificationService { + + private ImageClassificationClient imageClassificationClient; + private File repository = new File(System.getProperty("java.io.tmpdir")); + + + public void classifyImages(Document classifiedDoc) { + + classifiedDoc.getPages().forEach(page -> { + page.getImages().forEach(image -> { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + ImageIO.write(image.getImage(), "png", baos); + String fileName = UUID.randomUUID().toString() + ".png"; + FileItem fileItem = new DiskFileItem(fileName, "image/png", true, fileName, 100000000, repository); + MultipartFile multipartFile = new CommonsMultipartFile(fileItem); + ImageClassificationResponse response = imageClassificationClient.classify(multipartFile); + image.setImageType(ImageType.valueOf(response.getCategory())); + fileItem.delete(); + } catch (IOException e) { + e.printStackTrace(); + } + }); + }); + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 6b5278ae..8f526e8d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -63,7 +63,7 @@ public class RedactionLogCreatorService { classifiedDoc.getRedactionLogEntities().addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, ruleSetId)); } - if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) { + if (!classifiedDoc.getPages().get(page - 1).getImages().isEmpty()) { addImageEntries(classifiedDoc, page, ruleSetId); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java index 0e6f43f3..27c3bd52 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java @@ -15,7 +15,6 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService; import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService; import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper; -import com.iqser.red.service.redaction.v1.server.parsing.model.ParsedElements; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings; @@ -57,19 +56,10 @@ public class PdfSegmentationService { int rotation = pdPage.getRotation(); boolean isRotated = rotation != 0 && rotation != 360; - ParsedElements parsedElements = ParsedElements.builder() - .rulings(stripper.getRulings()) - .sequences(stripper.getTextPositionSequences()) - .imageBounds(stripper.getImageBounds()) - .minCharWidth(stripper.getMinCharWidth()) - .maxCharWidth(stripper.getMaxCharWidth()) - .landscape(isLandscape) - .rotated(isRotated) - .build(); + CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper + .getMaxCharHeight()); - CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight()); - - Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings + Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings .getVertical()); page.setRotation(rotation); @@ -77,11 +67,11 @@ public class PdfSegmentationService { buildPageStatistics(page); - page.setLandscape(parsedElements.isLandscape() || parsedElements.isRotated()); + page.setLandscape(isLandscape || isRotated); page.setPageNumber(pageNumber); increaseDocumentStatistics(page, document); - page.setImageBounds(parsedElements.getImageBounds()); + page.setImages(stripper.getImages()); pages.add(page); }