Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-2536: Treat \t same as whitespace

Browse Source
This commit is contained in:
Dominique Eifländer 2021-10-28 12:42:18 +02:00
parent 699cad7666
commit ca04c78724
4 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java
View File

@ -295,7 +295,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i) if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
.getUnicode() .getUnicode()
.equals("\u00A0"))) { .equals("\u00A0") || textPositions.get(i)
.getUnicode()
.equals("\t"))) {
startIndex++; startIndex++;
continue; continue;
} }
@ -305,7 +307,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
List<TextPosition> sublist = textPositions.subList(startIndex, i); List<TextPosition> sublist = textPositions.subList(startIndex, i);
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0) if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
.getUnicode() .getUnicode()
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) { .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0).getUnicode().equals("\t")))) {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
} }
startIndex = i; startIndex = i;
@ -316,7 +318,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
List<TextPosition> sublist = textPositions.subList(startIndex, i); List<TextPosition> sublist = textPositions.subList(startIndex, i);
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0) if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
.getUnicode() .getUnicode()
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) { .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0).getUnicode().equals("\t")))) {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
} }
startIndex = i; startIndex = i;
@ -324,11 +326,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i) if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
.getUnicode() .getUnicode()
.equals("\u00A0")) && i <= textPositions.size() - 2) { .equals("\u00A0") || textPositions.get(i)
.getUnicode()
.equals("\t")) && i <= textPositions.size() - 2) {
List<TextPosition> sublist = textPositions.subList(startIndex, i); List<TextPosition> sublist = textPositions.subList(startIndex, i);
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0) if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
.getUnicode() .getUnicode()
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) { .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0).getUnicode().equals("\t")))) {
// Remove false sequence ends (whitespaces) // Remove false sequence ends (whitespaces)
if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0) if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
@ -347,13 +351,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size()); List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1) if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
.getUnicode() .getUnicode()
.equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) { .equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0") || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
sublist = sublist.subList(0, sublist.size() - 1); sublist = sublist.subList(0, sublist.size() - 1);
} }
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0) if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
.getUnicode() .getUnicode()
.equals("\u00A0")))) { .equals("\u00A0") || sublist.get(0)
.getUnicode()
.equals("\t")))) {
if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0) if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
.getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) { .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
for (TextPosition t : sublist) { for (TextPosition t : sublist) {

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -656,7 +656,7 @@ public class RedactionIntegrationTest {
public void redactionTest() throws IOException { public void redactionTest() throws IOException {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/new/VV-919901.pdf");
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
request.setExcludedPages(Set.of(1)); request.setExcludedPages(Set.of(1));

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt
View File

@ -2,3 +2,4 @@ Amendment 1
Report Number: 33168 Report Number: 33168
Page Page
Report Number: BFI0714 Report Number: BFI0714
Tesh Consultants International

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/VV-919901.pdf Normal file
View File

Binary file not shown.