Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #42: RED-318

Browse Source 
Merge in RED/redaction-service from RED-318 to master

* commit 'f5790bccab3fb8dce66aeb9fbe501655ae6b86c3':
  Log entries and manual redaction require one position per line
  Annotate line by line
  Refactor multiple positions to one
  Merge rectangles for single annotation/redaction log entry
This commit is contained in:
Thierry Goeckel 2020-09-29 14:15:21 +02:00
commit c9516205fe
3 changed files with 61 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java
View File

@ -9,9 +9,7 @@ import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import lombok.Data;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
@Data
@RequiredArgsConstructor
@ -19,10 +17,6 @@ public class TextPositionSequence implements CharSequence {
private List<TextPosition> textPositions = new ArrayList<>();
@Getter
@Setter
private float[] annotationColor;
private final int page;
@ -194,22 +188,18 @@ public class TextPositionSequence implements CharSequence {
public Rectangle getRectangle() {
float height = textPositions.get(0).getHeightDir() + 2;
float height = getTextHeight();
float posXInit;
float posXInit = getX1();
float posXEnd;
float posYInit;
float posYEnd;
if (textPositions.get(0).getRotation() == 90) {
posXEnd = textPositions.get(0).getYDirAdj() + 2;
posXInit = textPositions.get(0).getYDirAdj() - height;
posYInit = textPositions.get(0).getXDirAdj();
posYInit = getY1();
posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
} else {
posXInit = textPositions.get(0).getXDirAdj();
posXEnd = textPositions.get(textPositions.size() - 1)
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
@ -220,4 +210,5 @@ public class TextPositionSequence implements CharSequence {
return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -27,9 +27,7 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class EntityRedactionService {

96
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java
View File

@ -2,9 +2,11 @@ package com.iqser.red.service.redaction.v1.server.visualization.service;
import java.awt.Color;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.pdfbox.pdmodel.PDDocument;
@ -16,6 +18,7 @@ import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
import org.apache.pdfbox.text.TextPosition;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
@ -35,9 +38,7 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class AnnotationHighlightService {
@ -45,7 +46,8 @@ public class AnnotationHighlightService {
private final DictionaryService dictionaryService;
public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction, ManualRedactions manualRedactions) throws IOException {
public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction,
ManualRedactions manualRedactions) throws IOException {
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
@ -83,7 +85,8 @@ public class AnnotationHighlightService {
}
private void addAnnotations(PDPage pdPage, Document classifiedDoc, boolean flatRedaction, ManualRedactions manualRedactions, int page) throws IOException {
private void addAnnotations(PDPage pdPage, Document classifiedDoc, boolean flatRedaction,
ManualRedactions manualRedactions, int page) throws IOException {
List<PDAnnotation> annotations = pdPage.getAnnotations();
@ -107,11 +110,15 @@ public class AnnotationHighlightService {
redactionLogEntry.setManual(true);
}
for (TextPositionSequence textPositions : entityPositionSequence.getSequences()) {
Rectangle rectangle = textPositions.getRectangle();
redactionLogEntry.getPositions().add(rectangle);
annotations.add(createAnnotation(rectangle, entityPositionSequence.getId(), createAnnotationContent(entity), getColor(entity), !flatRedaction && !isHint(entity)));
if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
for (Rectangle rectangle : getRectanglesPerLine(entityPositionSequence.getSequences()
.stream()
.flatMap(seq -> seq.getTextPositions().stream())
.collect(Collectors.toList()), page)) {
redactionLogEntry.getPositions().add(rectangle);
annotations.add(createAnnotation(rectangle, entityPositionSequence.getId(),
createAnnotationContent(entity), getColor(entity), !flatRedaction && !isHint(entity)));
}
}
redactionLogEntry.setId(entityPositionSequence.getId());
}
@ -120,7 +127,29 @@ public class AnnotationHighlightService {
}
private void addManualAnnotations(PDPage pdPage, Document classifiedDoc, ManualRedactions manualRedactions, int page) throws IOException {
private List<Rectangle> getRectanglesPerLine(List<TextPosition> textPositions, int page) {
List<Rectangle> rectangles = new ArrayList<>();
float y = textPositions.get(0).getYDirAdj();
int startIndex = 0;
for (int i = 1; i < textPositions.size(); i++) {
float yDirAdj = textPositions.get(i).getYDirAdj();
if (yDirAdj != y) {
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, i), page).getRectangle());
y = yDirAdj;
startIndex = i;
}
}
if (startIndex != textPositions.size() - 1) {
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
}
return rectangles;
}
private void addManualAnnotations(PDPage pdPage, Document classifiedDoc, ManualRedactions manualRedactions,
int page) throws IOException {
if (manualRedactions == null) {
return;
@ -143,8 +172,8 @@ public class AnnotationHighlightService {
foundOnPage = true;
PDAnnotationTextMarkup highlight = createAnnotation(rectangle, id, createAnnotationContent(manualRedactionEntry), getColor(manualRedactionEntry
.getType()), true);
PDAnnotationTextMarkup highlight = createAnnotation(rectangle, id,
createAnnotationContent(manualRedactionEntry), getColor(manualRedactionEntry.getType()), true);
annotations.add(highlight);
redactionLogEntry.getPositions().add(rectangle);
@ -188,7 +217,8 @@ public class AnnotationHighlightService {
}
private PDAnnotationTextMarkup createAnnotation(Rectangle rectangle, String id, String content, float[] color, boolean popup) {
private PDAnnotationTextMarkup createAnnotation(Rectangle rectangle, String id, String content, float[] color,
boolean popup) {
PDAnnotationTextMarkup annotation = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
annotation.constructAppearances();
@ -206,25 +236,15 @@ public class AnnotationHighlightService {
private String createAnnotationContent(Entity entity) {
return new StringBuilder().append("\nRule ")
.append(entity.getMatchedRule())
.append(" matched")
.append("\n\n")
.append(entity.getRedactionReason())
.append("\n\nIn Section : \"")
.append(entity.getHeadline())
.append("\"")
.toString();
return "\nRule " + entity.getMatchedRule() + " matched" + "\n\n" + entity.getRedactionReason() + "\n\nIn " +
"Section : \"" + entity
.getHeadline() + "\"";
}
private String createAnnotationContent(ManualRedactionEntry entry) {
return new StringBuilder().append("\nManual Redaction")
.append("\n\nIn Section : \"")
.append(entry.getSection())
.append("\"")
.toString();
return "\nManual Redaction" + "\n\nIn Section : \"" + entry.getSection() + "\"";
}
@ -244,7 +264,8 @@ public class AnnotationHighlightService {
// quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
// of the area to be highlighted
return new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getTopLeft()
.getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rectangle.getTopLeft().getX(), rectangle.getTopLeft()
.getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rectangle.getTopLeft().getX(),
rectangle.getTopLeft()
.getY() + rectangle.getHeight(), rectangle.getTopLeft()
.getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
}
@ -255,10 +276,7 @@ public class AnnotationHighlightService {
if (!entity.isRedaction()) {
return false;
}
if (isHint(entity)) {
return false;
}
return true;
return !isHint(entity);
}
@ -286,20 +304,19 @@ public class AnnotationHighlightService {
private boolean isHint(Entity entity) {
List<String> hintTypes = dictionaryService.getHintTypes();
if (CollectionUtils.isNotEmpty(hintTypes) && hintTypes.contains(entity.getType())) {
return true;
}
return false;
return CollectionUtils.isNotEmpty(hintTypes) && hintTypes.contains(entity.getType());
}
private void drawSectionFrames(PDDocument document, Document classifiedDoc, boolean flatRedaction, PDPage pdPage, int page) throws IOException {
private void drawSectionFrames(PDDocument document, Document classifiedDoc, boolean flatRedaction, PDPage pdPage,
int page) throws IOException {
if (flatRedaction) {
return;
}
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage,
PDPageContentStream.AppendMode.APPEND, true);
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
@ -348,7 +365,8 @@ public class AnnotationHighlightService {
if (cell != null) {
contentStream.setLineWidth(0.5f);
contentStream.setStrokingColor(Color.CYAN);
contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell
contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(),
(float) cell
.getHeight());
contentStream.stroke();
}