Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #140: Improved redaction performance

Browse Source 
Merge in RED/redaction-service from improved-redaction-performance to master

* commit '42fcea85d30f914fe02e304e69a551da409c6c3b':
  set image type on error
  proper error handling for image clasification
This commit is contained in:
Timo Bejan 2021-04-18 11:39:27 +02:00
commit 674ebf8eb7
3 changed files with 68 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImageClassificationService.java
View File

@ -12,7 +12,6 @@ import org.springframework.stereotype.Service;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException;
@Slf4j @Slf4j
@Service @Service
@ -35,8 +34,9 @@ public class ImageClassificationService {
var mockFile = new MockMultipartFile("file", "Image.png", "image/png", baos.toByteArray()); var mockFile = new MockMultipartFile("file", "Image.png", "image/png", baos.toByteArray());
ImageClassificationResponse response = imageClassificationClient.classify(mockFile); ImageClassificationResponse response = imageClassificationClient.classify(mockFile);
image.setImageType(ImageType.valueOf(response.getCategory())); image.setImageType(ImageType.valueOf(response.getCategory()));
} catch (IOException e) { } catch (Exception e) {
log.error("Could not classify image", e); log.error("Could not classify image", e);
image.setImageType(ImageType.OTHER);
} }
log.info("Image classification took: " + (System.currentTimeMillis() - start)); log.info("Image classification took: " + (System.currentTimeMillis() - start));

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java
View File

@ -127,7 +127,8 @@ public class PdfSegmentationService {
public Document parseDocument(InputStream documentInputStream) throws IOException { public Document parseDocument(InputStream documentInputStream) throws IOException {
PDDocument pdDocument = null;
try {
//create tempFile //create tempFile
File tempFile = File.createTempFile("document", ".pdf"); File tempFile = File.createTempFile("document", ".pdf");
IOUtils.copy(documentInputStream, new FileOutputStream(tempFile)); IOUtils.copy(documentInputStream, new FileOutputStream(tempFile));
@ -137,7 +138,7 @@ public class PdfSegmentationService {
List<Page> pages = new ArrayList<>(); List<Page> pages = new ArrayList<>();
PDDocument pdDocument = reinitializePDDocument(tempFile, null); pdDocument = reinitializePDDocument(tempFile, null);
long pageCount = pdDocument.getNumberOfPages(); long pageCount = pdDocument.getNumberOfPages();
for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) { for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
@ -201,6 +202,11 @@ public class PdfSegmentationService {
tempFile.delete(); tempFile.delete();
return document; return document;
} finally {
if (pdDocument != null) {
pdDocument.close();
}
}
} }
private PDDocument reinitializePDDocument(File tempFile, PDDocument pdDocument) throws IOException { private PDDocument reinitializePDDocument(File tempFile, PDDocument pdDocument) throws IOException {

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -19,7 +19,6 @@ import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.kie.api.KieServices; import org.kie.api.KieServices;
@ -452,7 +451,6 @@ public class RedactionIntegrationTest {
@Test @Test
@Ignore
public void testLargeScannedFileOOM() { public void testLargeScannedFileOOM() {
AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf"); AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
MemoryStats.printMemoryStats(); MemoryStats.printMemoryStats();