From f49782438461ec34e6e8c0ccf88b3017755e593d Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Thu, 16 Dec 2021 14:08:05 +0100 Subject: [PATCH] RED-3067 Bugfix to show redactions on rotated pages correctly and fixed getting tmp directory --- .../parsing/model/TextPositionSequence.java | 6 +- .../service/RedactionLogCreatorService.java | 4 +- .../v1/server/RedactionIntegrationTest.java | 71 +++++++++---------- .../v1/server/redaction/utils/OsUtils.java | 22 ++++++ .../PdfSegmentationServiceTest.java | 8 +-- 5 files changed, 62 insertions(+), 49 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OsUtils.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java index 2175b4fd..518aefa2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java @@ -254,7 +254,7 @@ public class TextPositionSequence implements CharSequence { posYInit = getY1(); posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4; - } else if (textPositions.get(0).getRotation() == 270 && textPositions.size() > 1) { + } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 270.0f) { posYInit = textPositions.get(0).getPageHeight() - getX1(); posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0) .getWidth() - textPositions.get(textPositions.size() - 1).getWidth() - 1; @@ -262,11 +262,11 @@ public class TextPositionSequence implements CharSequence { posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1) .getYDirAdj() + height; - } else if (textPositions.get(0).getRotation() == 270 && textPositions.size() == 1) { + } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 0.0f) { posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2; posYEnd = posYInit + 1; posXInit = textPositions.get(0).getXDirAdj(); - posXEnd = posXInit + textPositions.get(0).getWidthDirAdj() + 0.1f; + posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f; } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 270f) { posYInit = textPositions.get(0).getPageHeight() - getX1(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 6075cf74..6be2e194 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -140,14 +140,14 @@ public class RedactionLogCreatorService { } else { float x = textPositions.get(0).getXDirAdj(); float y = textPositions.get(0).getYDirAdj(); - float width = textPositions.get(0).getWidth(); + float width = textPositions.get(0).getWidthDirAdj(); float height = textPositions.get(0).getHeightDir(); int startIndex = 0; for (int i = 1; i < textPositions.size(); i++) { float xDirAdj = textPositions.get(i).getXDirAdj(); float yDirAdj = textPositions.get(i).getYDirAdj(); - float widthDir = textPositions.get(i).getWidth(); + float widthDir = textPositions.get(i).getWidthDirAdj(); float heightDir = textPositions.get(i).getHeightDir(); if (!(isCharInSameLine(y, yDirAdj, height, heightDir) && isCharClose(x, xDirAdj, width))) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 5c1f24e0..292bfc87 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -24,7 +24,6 @@ import java.util.UUID; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -50,7 +49,6 @@ import org.springframework.test.context.junit4.SpringRunner; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval; @@ -84,6 +82,7 @@ import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.memory.MemoryStats; import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; +import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -596,7 +595,7 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated3.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated3.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } long rstart = System.currentTimeMillis(); @@ -623,13 +622,15 @@ public class RedactionIntegrationTest { var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList()); + var values = redactionLog.getRedactionLogEntry() + .stream() + .map(RedactionLogEntry::getValue) + .collect(Collectors.toList()); assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname"); } - @Test @SneakyThrows public void testIgnoreHint() { @@ -664,17 +665,22 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - - var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry() + .stream() + .filter(re -> re.getType().equalsIgnoreCase("CBI_Address")) + .findAny() + .get(); assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse(); - var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry() + .stream() + .filter(re -> re.getType().equalsIgnoreCase("CBI_Address")) + .findAny() + .get(); assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue(); } - - @Test @Ignore public void noExceptionShouldBeThrownForAnyFiles() throws IOException { @@ -732,7 +738,7 @@ public class RedactionIntegrationTest { public void redactionTestSeparatedRedaction() throws IOException { String fileName = "scanned/VV-380943_page38.pdf"; - String outputFileName = getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf"; + String outputFileName = OsUtils.getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf"; long start = System.currentTimeMillis(); ClassPathResource pdfFileResource = new ClassPathResource(fileName); @@ -756,7 +762,7 @@ public class RedactionIntegrationTest { System.out.println("first analysis duration: " + (end - start)); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Test.json")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Test.json")) { fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID))); } @@ -841,15 +847,15 @@ public class RedactionIntegrationTest { redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - System.out.println("hi"); + System.out.println("Output file:" + outputFileName); } @Test public void redactionTest() throws IOException { - String fileName = "files/new/VV-919901.pdf"; - String outputFileName = getTemporaryDirectory() + "/Annotated.pdf"; + String fileName = "files/new/VV-511309.pdf"; + String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; long start = System.currentTimeMillis(); ClassPathResource pdfFileResource = new ClassPathResource(fileName); @@ -873,7 +879,7 @@ public class RedactionIntegrationTest { System.out.println("first analysis duration: " + (end - start)); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Test.json")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Test.json")) { fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID))); } @@ -958,7 +964,7 @@ public class RedactionIntegrationTest { redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - System.out.println("hi"); + System.out.println("Output file:" + outputFileName); } @@ -977,7 +983,7 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } long end = System.currentTimeMillis(); @@ -1071,7 +1077,7 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } long end = System.currentTimeMillis(); @@ -1097,7 +1103,7 @@ public class RedactionIntegrationTest { RedactionResult result = redactionController.classify(redactionRequest); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Classified.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) { fileOutputStream.write(result.getDocument()); } } @@ -1119,7 +1125,7 @@ public class RedactionIntegrationTest { RedactionResult result = redactionController.sections(redactionRequest); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Sections.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Sections.pdf")) { fileOutputStream.write(result.getDocument()); } } @@ -1141,7 +1147,7 @@ public class RedactionIntegrationTest { RedactionResult result = redactionController.htmlTables(redactionRequest); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Tables.html")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) { fileOutputStream.write(result.getDocument()); } } @@ -1163,7 +1169,7 @@ public class RedactionIntegrationTest { RedactionResult result = redactionController.htmlTables(redactionRequest); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Tables.html")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) { fileOutputStream.write(result.getDocument()); } } @@ -1233,7 +1239,7 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } long end = System.currentTimeMillis(); @@ -1366,12 +1372,13 @@ public class RedactionIntegrationTest { .manualRedactions(manualRedactions) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } } + @Test public void testExpandByRegEx() throws IOException { @@ -1387,7 +1394,7 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } long end = System.currentTimeMillis(); @@ -1465,7 +1472,7 @@ public class RedactionIntegrationTest { .manualRedactions(manualRedactions) .build()); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) { + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); } @@ -1513,14 +1520,4 @@ public class RedactionIntegrationTest { } - - private static String getTemporaryDirectory() { - -// String tmpdir = System.getProperty("java.io.tmpdir"); -// if (StringUtils.isNotBlank(tmpdir)) { -// return tmpdir; -// } - return "/tmp"; - } - } \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OsUtils.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OsUtils.java new file mode 100644 index 00000000..c9554259 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OsUtils.java @@ -0,0 +1,22 @@ +package com.iqser.red.service.redaction.v1.server.redaction.utils; + +import org.apache.commons.lang3.StringUtils; + +public class OsUtils { + + public static boolean isWindows() { + + return StringUtils.containsIgnoreCase(System.getProperty("os.name"), "Windows"); + } + + + public static String getTemporaryDirectory() { + + String tmpdir = System.getProperty("java.io.tmpdir"); + if (isWindows() && StringUtils.isNotBlank(tmpdir)) { + return tmpdir; + } + return "/tmp"; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java index e24dcbbf..f773d281 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import static com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils.getTemporaryDirectory; import static org.assertj.core.api.Assertions.assertThat; @@ -240,12 +241,5 @@ public class PdfSegmentationServiceTest { .equals(firstTableHeaderCells))).isTrue(); } - private static String getTemporaryDirectory() { - String tmpdir = System.getProperty("java.io.tmpdir"); - if (StringUtils.isNotBlank(tmpdir)) { - return tmpdir; - } - return "/tmp"; - } }