Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6369: Rules Refactor

Browse Source 
*removed manualredaction handling for further analysis of how it actually works
This commit is contained in:
Kilian Schuettler 2023-04-17 16:34:08 +02:00
parent 86b5059d77
commit 5628db7a36
11 changed files with 109 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java
View File

@ -1,9 +1,13 @@
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
import java.awt.geom.Rectangle2D;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.google.common.hash.Hashing;
@ -12,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@ -50,7 +55,7 @@ public class EntityNode {
CharSequence textAfter;
@Builder.Default
Set<PageNode> pages = new HashSet<>();
List<EntityPosition> entityPositionsPerPage;
List<RedactionPosition> redactionPositionsPerPage;
@Builder.Default
List<SemanticNode> intersectingNodes = new LinkedList<>();
SemanticNode deepestFullyContainingNode;
@ -93,13 +98,30 @@ public class EntityNode {
}
public List<EntityPosition> getEntityPositionsPerPage() {
public List<RedactionPosition> getRedactionPositionsPerPage() {
if (entityPositionsPerPage == null || entityPositionsPerPage.isEmpty()) {
entityPositionsPerPage = deepestFullyContainingNode.buildTextBlock().getEntityPositionsPerPage(boundary);
if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) {
Map<PageNode, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.buildTextBlock().getEntityPositionsPerPage(boundary);
PageNode firstPage = rectanglesPerLinePerPage.keySet()
.stream()
.min(Comparator.comparingInt(PageNode::getNumber))
.orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList());
redactionPositionsPerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildRedactionPosition(firstPage, id, entry)).toList();
}
return redactionPositionsPerPage;
}
return entityPositionsPerPage;
private static RedactionPosition buildRedactionPosition(PageNode firstPage, String id, Map.Entry<PageNode, List<Rectangle2D>> entry) {
if (entry.getKey().equals(firstPage)) {
return new RedactionPosition(id, entry.getKey(), entry.getValue());
} else {
return new RedactionPosition(id + "-" + entry.getKey().getNumber(), entry.getKey(), entry.getValue());
}
}
@ -141,7 +163,7 @@ public class EntityNode {
public boolean matchesResizedAnnotationId(String resizedAnnotationId) {
return getEntityPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(resizedAnnotationId));
return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(resizedAnnotationId));
}

39
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java
View File

@ -1,39 +0,0 @@
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
import java.awt.geom.Rectangle2D;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.google.common.hash.Hashing;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import lombok.AccessLevel;
import lombok.Builder;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@FieldDefaults(level = AccessLevel.PRIVATE)
public class EntityPosition {
PageNode pageNode;
List<Rectangle2D> rectanglePerLine;
public String getId() {
return String.valueOf(hashCode());
}
@Override
public int hashCode() {
StringBuilder sb = new StringBuilder();
sb.append(pageNode.getNumber());
rectanglePerLine.forEach(r -> sb.append(r.getX()).append(r.getY()).append(r.getWidth()).append(r.getHeight()));
return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode();
}
}

22
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/RedactionPosition.java Normal file
View File

@ -0,0 +1,22 @@
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
import java.awt.geom.Rectangle2D;
import java.util.List;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class RedactionPosition {
final String id;
PageNode pageNode;
List<Rectangle2D> rectanglePerLine;
}

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java
View File

@ -3,10 +3,11 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
import static java.lang.String.format;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
@ -112,15 +113,16 @@ public class AtomicTextBlock implements TextBlock {
}
public List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary) {
public Map<PageNode, List<Rectangle2D>> getEntityPositionsPerPage(Boundary stringBoundary) {
List<Rectangle2D> positionsPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList())
List<Rectangle2D> rectanglesPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList())
.stream()
.map(this::getPositions)
.map(RectangleTransformations::rectangleUnion)
.toList();
return List.of(EntityPosition.builder().rectanglePerLine(positionsPerLine).pageNode(page).build());
Map<PageNode, List<Rectangle2D>> rectanglePerLinePerPage = new HashMap<>();
rectanglePerLinePerPage.put(page, rectanglesPerLine);
return rectanglePerLinePerPage;
}

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java
View File

@ -1,17 +1,14 @@
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
import static java.lang.String.format;
import static java.util.stream.Collectors.flatMapping;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import lombok.AccessLevel;
@ -139,7 +136,7 @@ public class ConcatenatedTextBlock implements TextBlock {
@Override
public List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary) {
public Map<PageNode, List<Rectangle2D>> getEntityPositionsPerPage(Boundary stringBoundary) {
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
@ -148,29 +145,28 @@ public class ConcatenatedTextBlock implements TextBlock {
}
AtomicTextBlock firstTextBlock = textBlocks.get(0);
List<EntityPosition> positions = new LinkedList<>(firstTextBlock.getEntityPositionsPerPage(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
Map<PageNode, List<Rectangle2D>> rectanglesPerLinePerPage = firstTextBlock.getEntityPositionsPerPage(new Boundary(stringBoundary.start(),
firstTextBlock.getBoundary().end()));
for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
positions.addAll(textBlock.getEntityPositionsPerPage(textBlock.getBoundary()));
rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, textBlock.getEntityPositionsPerPage(textBlock.getBoundary()));
}
AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
positions.addAll(lastTextBlock.getEntityPositionsPerPage(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, lastTextBlock.getEntityPositionsPerPage(lastTextBlock.getBoundary()));
return mergeEntityPositionsWithSamePageNode(positions);
return rectanglesPerLinePerPage;
}
private List<EntityPosition> mergeEntityPositionsWithSamePageNode(List<EntityPosition> positions) {
Map<PageNode, List<Rectangle2D>> entityPositionsPerPage = positions.stream().collect(//
groupingBy(EntityPosition::getPageNode, //
flatMapping(entityPosition -> entityPosition.getRectanglePerLine().stream(), toList())));
return entityPositionsPerPage.entrySet().stream()//
.map(entry -> EntityPosition.builder().pageNode(entry.getKey()).rectanglePerLine(entry.getValue()).build())//
.toList();
private Map<PageNode, List<Rectangle2D>> mergeEntityPositionsWithSamePageNode(Map<PageNode, List<Rectangle2D>> map1, Map<PageNode, List<Rectangle2D>> map2) {
Map<PageNode, List<Rectangle2D>> mergedMap = new HashMap<>(map1);
map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, rectangles, (rects1, rects2) -> {
rects1.addAll(rects2);
return rects1;
}));
return mergedMap;
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java
View File

@ -6,11 +6,11 @@ import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
public interface TextBlock extends CharSequence {
@ -39,7 +39,7 @@ public interface TextBlock extends CharSequence {
List<Rectangle2D> getPositions(Boundary stringBoundary);
List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary);
Map<PageNode, List<Rectangle2D>> getEntityPositionsPerPage(Boundary stringBoundary);
int numberOfLines();

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -15,7 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@ -52,20 +52,20 @@ public class RedactionLogCreatorService {
// Duplicates can exist due table extraction columns over multiple rows.
for (EntityPosition entityPosition : entityNode.getEntityPositionsPerPage()) {
for (RedactionPosition redactionPosition : entityNode.getRedactionPositionsPerPage()) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entityNode, dossierTemplateId);
if (processedIds.contains(entityPosition.getId())) {
if (processedIds.contains(redactionPosition.getId())) {
continue;
}
processedIds.add(entityPosition.getId());
redactionLogEntry.setId(entityPosition.getId());
processedIds.add(redactionPosition.getId());
redactionLogEntry.setId(redactionPosition.getId());
List<Rectangle> rectanglesPerLine = entityPosition.getRectanglePerLine()
List<Rectangle> rectanglesPerLine = redactionPosition.getRectanglePerLine()
.stream()
.map(rectangle2D -> toRectangle(rectangle2D, entityPosition.getPageNode().getNumber()))
.map(rectangle2D -> toRectangle(rectangle2D, redactionPosition.getPageNode().getNumber()))
.toList();
redactionLogEntry.setPositions(rectanglesPerLine);
@ -79,7 +79,7 @@ public class RedactionLogCreatorService {
private RedactionLogEntry createRedactionLogEntry(EntityNode entity, String dossierTemplateId) {
Set<String> referenceIds = new HashSet<>();
entity.getReferences().forEach(ref -> ref.getEntityPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
entity.getReferences().forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java
View File

@ -3,9 +3,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
import java.awt.geom.Rectangle2D;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Set;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
@ -28,12 +30,23 @@ public final class IdBuilder {
}
public String buildId(Set<PageNode> pages, List<Rectangle2D> rectanglesPerLine) {
StringBuilder sb = new StringBuilder();
pages.forEach(pageNode -> sb.append(pageNode.getNumber()));
rectanglesPerLine.forEach(rectangle2D -> sb.append(rectangle2D.getX()).append(rectangle2D.getY()).append(rectangle2D.getWidth()).append(rectangle2D.getHeight()));
return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
}
public String buildId(Rectangle2D rectangle2D, int page) {
return hashFunction.hashString("x" + rectangle2D.getX() + "y" + rectangle2D.getY() + "h" + rectangle2D.getHeight() + "w" + rectangle2D.getWidth() + "p" + page,
StandardCharsets.UTF_8).toString();
}
public String buildId(RedRectangle2D rectangle2D, int page) {
return hashFunction.hashString("x" + rectangle2D.getX() + "y" + rectangle2D.getY() + "h" + rectangle2D.getHeight() + "w" + rectangle2D.getWidth() + "p" + page,

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java
View File

@ -217,7 +217,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
.stream()
.filter(entityNode -> !entityNode.isRemoved())
.filter(EntityNode::isRedaction)
.flatMap(entityNode -> entityNode.getEntityPositionsPerPage().stream())
.flatMap(entityNode -> entityNode.getRedactionPositionsPerPage().stream())
.filter(entityPosition -> entityPosition.getPageNode().equals(page))
.flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
.toList();
@ -231,7 +231,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
.stream()
.filter(entityNode -> !entityNode.isRemoved())
.filter(entityNode -> !entityNode.isRedaction())
.flatMap(entityNode -> entityNode.getEntityPositionsPerPage().stream())
.flatMap(entityNode -> entityNode.getRedactionPositionsPerPage().stream())
.filter(entityPosition -> entityPosition.getPageNode().equals(page))
.flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
.toList();

12
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionTest.java Normal file
View File

@ -0,0 +1,12 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import org.junit.jupiter.api.Test;
public class ManualResizeRedactionTest extends BuildDocumentGraphTest {
@Test
public void manualResizeRedactionTest() {
}
}

15
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -32,19 +32,6 @@ query "getFileAttributes"
$fileAttribute: FileAttribute()
end
// --------------------------------------- apply resize redactions -------------------------------------------------------------------
rule "apply resize redactions"
salience 200
no-loop true
when
$resizeRedaction: ManualResizeRedaction($id: annotationId)
$resizedEntity: EntityNode(matchesResizedAnnotationId($id))
then
$resizedEntity.setResized(true);
update($resizedEntity);
end
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "merge intersecting Entities of same type"
@ -57,9 +44,9 @@ rule "merge intersecting Entities of same type"
$first.removeFromGraph();
$second.removeFromGraph();
EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
insert(mergedEntity);
retract($first);
retract($second);
insert(mergedEntity);
end
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"