From d601942b7a57f6dcfe7027b3ee9e01628d4beb80 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 25 May 2023 17:05:36 +0200 Subject: [PATCH] RED-6009: Document Tree Structure * some refactoring like renaming * added migration poc --- .../adapter/RedactionLogEntryAdapter.java | 164 +++++ .../model/AbstractPageBlock.java | 6 +- .../model/ClassificationFooter.java | 4 +- .../model/ClassificationHeader.java | 4 +- .../classification/model/PageBlockType.java | 8 +- .../classification/model/table/Cell.java | 10 +- .../model/table/TablePageBlock.java | 16 +- ...ationTextBlock.java => TextPageBlock.java} | 21 +- .../model/text/UnclassifiedText.java | 2 +- .../service/BlockificationService.java | 27 +- .../service/BodyTextFrameService.java | 10 +- .../service/ClassificationService.java | 8 +- .../service/PdfSegmentationService.java | 8 +- .../service/SectionsBuilderService.java | 16 +- .../service/TableExtractionService.java | 8 +- .../classification/utils/PositionUtils.java | 14 +- .../document/data/DocumentTreeData.java | 12 +- .../data/mapper/DocumentGraphMapper.java | 95 ++- .../data/mapper/PropertiesMapper.java | 92 +-- .../factory/DocumentGraphFactory.java | 20 +- .../SearchTextWithTextPositionFactory.java | 38 +- .../document/factory/SectionNodeFactory.java | 35 +- .../document/factory/TableNodeFactory.java | 25 +- .../document/graph/Boundary.java | 28 +- .../document/graph/DocumentTree.java | 39 +- .../graph/entity/RedactionEntity.java | 43 +- .../graph/entity/RedactionPosition.java | 2 + .../document/graph/nodes/Document.java | 11 +- .../document/graph/nodes/Footer.java | 12 +- .../document/graph/nodes/Header.java | 12 +- .../document/graph/nodes/Headline.java | 12 +- .../document/graph/nodes/Image.java | 2 +- .../document/graph/nodes/ImageType.java | 14 +- .../document/graph/nodes/NodeType.java | 10 +- .../document/graph/nodes/Paragraph.java | 12 +- .../document/graph/nodes/Section.java | 11 +- .../document/graph/nodes/SemanticNode.java | 36 +- .../document/graph/nodes/Table.java | 18 +- .../document/graph/nodes/TableCell.java | 15 +- .../textblock/ConcatenatedTextBlock.java | 4 +- .../services/EntityCreationService.java | 51 +- .../services/EntityEnrichmentService.java | 4 +- .../ManualRedactionApplicationService.java | 10 +- .../utils/PdfVisualisationUtility.java | 10 +- .../document/utils/RectangleMapper.java | 59 -- .../utils/RectangleTransformations.java | 40 +- .../utils/RedactionSearchUtility.java | 23 +- .../utils/TextPositionOperations.java | 4 +- .../redaction/adapter/NerEntitiesAdapter.java | 6 +- .../model/dictionary/Dictionary.java | 5 +- .../redaction/service/AnalyzeService.java | 2 +- ...ManualRedactionSurroundingTextService.java | 17 +- .../service/RedactionLogCreatorService.java | 11 +- .../service/SectionFinderService.java | 2 +- .../service/SectionGridCreatorService.java | 6 +- .../v1/server/redaction/utils/IdBuilder.java | 16 +- .../redaction/utils/SeparatorUtils.java | 6 +- .../HeadlinesGoldStandardIntegrationTest.java | 2 +- .../v1/server/RedactionIntegrationTest.java | 42 +- .../v1/server/RedactionIntegrationV2Test.java | 4 +- .../v1/server/annotate/AnnotationService.java | 12 +- ...ocumentEntityInsertionIntegrationTest.java | 22 +- .../DocumentVisualizationIntegrationTest.java | 10 +- .../document/graph/MigrationPocTest.java | 162 ++++ .../DocumentPerformanceIntegrationTest.java | 4 +- .../adapter/NerEntitiesAdapterTest.java | 2 +- .../drools/manual_redaction_rules.drl | 3 +- .../resources/drools/prod_syngenta_new.drl | 3 +- .../src/test/resources/drools/rules.drl | 102 ++- .../test/resources/drools/rules_backup.drl | 697 ++++++++++++++++++ .../src/test/resources/drools/rules_v2.drl | 1 - .../files/migration/legacy_redactionlog.json | 1 + .../EFSA_sanitisation_GFL_v1/rules.drl | 4 +- 73 files changed, 1673 insertions(+), 594 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/{ClassificationTextBlock.java => TextPageBlock.java} (90%) delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleMapper.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/legacy_redactionlog.json diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java new file mode 100644 index 00000000..ac256585 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java @@ -0,0 +1,164 @@ +package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter; + +import static java.lang.String.format; +import static java.util.stream.Collectors.groupingBy; + +import java.awt.geom.Rectangle2D; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class RedactionLogEntryAdapter { + + private static final double MATCH_THRESHOLD = 1; + private final EntityCreationService entityCreationService; + + + public Stream toRedactionEntity(RedactionLog redactionLog, SemanticNode node) { + + List pageNumbers = redactionLog.getRedactionLogEntry().stream().flatMap(entry -> entry.getPositions().stream().map(Rectangle::getPage)).distinct().toList(); + if (!pageNumbers.stream().allMatch(node::isOnPage)) { + throw new IllegalArgumentException(format("SemanticNode %s does not contain these pages %s present in the redaction log", + node, + pageNumbers.stream().filter(pageNumber -> !node.isOnPage(pageNumber)).toList())); + } + Set entryValues = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).map(String::toLowerCase).collect(Collectors.toSet()); + SearchImplementation searchImplementation = new SearchImplementation(entryValues, true); + + Map> tempEntitiesByValue = findAllPossibleEntitiesAndGroupByValueIgnoringCase(node, searchImplementation); + + assert allValuesFound(tempEntitiesByValue, entryValues); + + List entities = redactionLog.getRedactionLogEntry() + .stream() + .map(entry -> findClosestRedactionEntity(entry, tempEntitiesByValue.get(entry.getValue().toLowerCase(Locale.ROOT)), node)) + .toList(); + tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(RedactionEntity::removeFromGraph); + return entities.stream(); + } + + + private static boolean allValuesFound(Map> entitiesByValue, Set entryValues) { + + return entitiesByValue.keySet().equals(entryValues); + } + + + private Map> findAllPossibleEntitiesAndGroupByValueIgnoringCase(SemanticNode node, SearchImplementation searchImplementation) { + + return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary()) + .stream() + .map(boundary -> entityCreationService.byBoundary(boundary, "temp", EntityType.ENTITY, node)) + .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT))); + } + + + private RedactionEntity findClosestRedactionEntity(RedactionLogEntry redactionLogEntry, List entitiesWithSameValue, SemanticNode node) { + + RedactionEntity closestEntity = entitiesWithSameValue.stream() + .filter(entity -> pagesMatch(entity, redactionLogEntry)) + .min(Comparator.comparingDouble(entity -> calculateMinDistance(redactionLogEntry, entity))) + .orElseThrow(() -> new NotFoundException(format("No entity with similar position found for %s", redactionLogEntry))); + + double distance = calculateMinDistance(redactionLogEntry, closestEntity); + if (distance > MATCH_THRESHOLD) { + throw new NotFoundException(format("Distance to closest found entity is %.2f for \n%s \n%s", + distance, + redactionLogEntry.getPositions(), + closestEntity.getRedactionPositionsPerPage())); + } + + return createCorrectEntity(redactionLogEntry, node, closestEntity); + } + + + private RedactionEntity createCorrectEntity(RedactionLogEntry redactionLogEntry, SemanticNode node, RedactionEntity closestEntity) { + + RedactionEntity correctEntity = entityCreationService.byBoundary(closestEntity.getBoundary(), + redactionLogEntry.getType(), + redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY, + node); + correctEntity.setLegalBasis(redactionLogEntry.getLegalBasis()); + correctEntity.setRedactionReason(redactionLogEntry.getReason()); + correctEntity.addMatchedRule(redactionLogEntry.getMatchedRule()); + correctEntity.setRedaction(redactionLogEntry.isRedacted()); + correctEntity.setDictionaryEntry(redactionLogEntry.isDictionaryEntry()); + correctEntity.setDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry()); + return correctEntity; + } + + + private static boolean pagesMatch(RedactionEntity entity, RedactionLogEntry redactionLogEntry) { + + Set entityPageNumbers = entity.getRedactionPositionsPerPage().stream().map(RedactionPosition::getPage).map(Page::getNumber).collect(Collectors.toSet()); + Set redactionLogEntryPageNumbers = redactionLogEntry.getPositions().stream().map(Rectangle::getPage).collect(Collectors.toSet()); + return entityPageNumbers.equals(redactionLogEntryPageNumbers); + } + + + private double calculateMinDistance(RedactionLogEntry redactionLogEntry, RedactionEntity entity) { + + if (redactionLogEntry.getPositions().size() != countRectangles(entity)) { + return Double.MAX_VALUE; + } + return redactionLogEntry.getPositions().stream().mapToDouble(redactionLogEntryRectangle -> calculateMinDistancePerRectangle(entity, redactionLogEntryRectangle)).sum(); + } + + + private static long countRectangles(RedactionEntity entity) { + + return entity.getRedactionPositionsPerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum(); + } + + + private double calculateMinDistancePerRectangle(RedactionEntity entity, Rectangle redactionLogEntryRectangle) { + + return entity.getRedactionPositionsPerPage() + .stream() + .filter(redactionPosition -> redactionPosition.getPage().getNumber() == redactionLogEntryRectangle.getPage()) + .map(RedactionPosition::getRectanglePerLine) + .flatMap(Collection::stream) + .mapToDouble(rectangle -> calculateDistance(rectangle, toRectangle2D(redactionLogEntryRectangle))) + .min() + .orElse(Double.MAX_VALUE); + } + + + private double calculateDistance(Rectangle2D rectangle, Rectangle2D rectangle2D) { + + return Math.abs(rectangle.getMinX() - rectangle2D.getMinX()) // + + Math.abs(rectangle.getMinY() - rectangle2D.getMinY()) // + + Math.abs(rectangle.getMaxX() - rectangle2D.getMaxX()) // + + Math.abs(rectangle.getMaxY() - rectangle2D.getMaxY()); + } + + + private Rectangle2D toRectangle2D(Rectangle rectangle) { + + return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight()); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java index 3cfcee36..a0d2caef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java @@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import lombok.AllArgsConstructor; import lombok.Data; @@ -35,11 +35,11 @@ public abstract class AbstractPageBlock { public boolean isHeadline() { - return this instanceof ClassificationTextBlock && this.getClassification() != null && this.getClassification().isHeadline(); + return this instanceof TextPageBlock && this.getClassification() != null && this.getClassification().isHeadline(); } - public boolean containsBlock(ClassificationTextBlock other) { + public boolean containsBlock(TextPageBlock other) { return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationFooter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationFooter.java index 6e14f087..221bda7f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationFooter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationFooter.java @@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m import java.util.List; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import lombok.AllArgsConstructor; import lombok.Data; @@ -11,6 +11,6 @@ import lombok.Data; @AllArgsConstructor public class ClassificationFooter { - private List textBlocks; + private List textBlocks; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationHeader.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationHeader.java index 17659b7f..53f1972a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationHeader.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationHeader.java @@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m import java.util.List; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import lombok.AllArgsConstructor; import lombok.Data; @@ -11,6 +11,6 @@ import lombok.Data; @AllArgsConstructor public class ClassificationHeader { - private List textBlocks; + private List textBlocks; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/PageBlockType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/PageBlockType.java index 16c52c0f..d2991efd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/PageBlockType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/PageBlockType.java @@ -5,6 +5,8 @@ public enum PageBlockType { H2, H3, H4, + H5, + H6, HEADER, FOOTER, TITLE, @@ -22,13 +24,15 @@ public enum PageBlockType { case 1 -> PageBlockType.H1; case 2 -> PageBlockType.H2; case 3 -> PageBlockType.H3; - default -> PageBlockType.H4; + case 4 -> PageBlockType.H4; + case 5 -> PageBlockType.H5; + default -> PageBlockType.H6; }; } public boolean isHeadline() { - return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4); + return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4) || this.equals(H5) || this.equals(H6); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Cell.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Cell.java index f8d66ffe..19b64a6a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Cell.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Cell.java @@ -5,7 +5,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; @@ -19,7 +19,7 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class Cell extends Rectangle { - private List textBlocks = new ArrayList<>(); + private List textBlocks = new ArrayList<>(); private List headerCells = new ArrayList<>(); @@ -36,7 +36,7 @@ public class Cell extends Rectangle { } - public void addTextBlock(ClassificationTextBlock textBlock) { + public void addTextBlock(TextPageBlock textBlock) { textBlocks.add(textBlock); } @@ -47,11 +47,11 @@ public class Cell extends Rectangle { StringBuilder sb = new StringBuilder(); - Iterator itty = textBlocks.iterator(); + Iterator itty = textBlocks.iterator(); TextPositionSequence previous = null; while (itty.hasNext()) { - ClassificationTextBlock textBlock = itty.next(); + TextPageBlock textBlock = itty.next(); for (TextPositionSequence word : textBlock.getSequences()) { if (previous != null) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java index 86df0526..8d8684ef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java @@ -7,13 +7,12 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.TreeMap; -import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.PageBlockType; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import lombok.Getter; import lombok.Setter; @@ -42,7 +41,6 @@ public class TablePageBlock extends AbstractPageBlock { maxY = area.getTop(); classification = PageBlockType.TABLE; this.rotation = rotation; - } @@ -236,8 +234,8 @@ public class TablePageBlock extends AbstractPageBlock { uniqueY.add(c.getTop()); }); - var sortedUniqueX = uniqueX.stream().sorted().collect(Collectors.toList()); - var sortedUniqueY = uniqueY.stream().sorted().collect(Collectors.toList()); + var sortedUniqueX = uniqueX.stream().sorted().toList(); + var sortedUniqueY = uniqueY.stream().sorted().toList(); Float prevY = null; for (Float y : sortedUniqueY) { @@ -251,9 +249,7 @@ public class TablePageBlock extends AbstractPageBlock { var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y)); var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst(); - if (intersectionCell.isPresent()) { - cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks()); - } + intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks())); if (cell.hasMinimumSize()) { row.add(cell); } @@ -292,7 +288,7 @@ public class TablePageBlock extends AbstractPageBlock { } if (column != null && column.getTextBlocks() != null) { boolean first = true; - for (ClassificationTextBlock textBlock : column.getTextBlocks()) { + for (TextPageBlock textBlock : column.getTextBlocks()) { if (!first) { sb.append("\n"); } @@ -324,7 +320,7 @@ public class TablePageBlock extends AbstractPageBlock { sb.append(i == 0 ? "\n" : "\n"); if (column != null && column.getTextBlocks() != null) { boolean first = true; - for (ClassificationTextBlock textBlock : column.getTextBlocks()) { + for (TextPageBlock textBlock : column.getTextBlocks()) { if (!first) { sb.append("
"); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextPageBlock.java similarity index 90% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextPageBlock.java index dbb07713..ca8cc8e2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextPageBlock.java @@ -17,7 +17,7 @@ import lombok.NoArgsConstructor; @Builder @Data @NoArgsConstructor -public class ClassificationTextBlock extends AbstractPageBlock { +public class TextPageBlock extends AbstractPageBlock { @Builder.Default private List sequences = new ArrayList<>(); @@ -25,8 +25,6 @@ public class ClassificationTextBlock extends AbstractPageBlock { @JsonIgnore private int rotation; - private int indexOnPage; - @JsonIgnore private String mostPopularWordFont; @@ -176,9 +174,8 @@ public class ClassificationTextBlock extends AbstractPageBlock { } - public ClassificationTextBlock(float minX, float maxX, float minY, float maxY, List sequences, int rotation, int indexOnPage) { + public TextPageBlock(float minX, float maxX, float minY, float maxY, List sequences, int rotation) { - this.indexOnPage = indexOnPage; this.minX = minX; this.maxX = maxX; this.minY = minY; @@ -188,23 +185,23 @@ public class ClassificationTextBlock extends AbstractPageBlock { } - public ClassificationTextBlock union(TextPositionSequence r) { + public TextPageBlock union(TextPositionSequence r) { - ClassificationTextBlock union = this.copy(); + TextPageBlock union = this.copy(); union.add(r); return union; } - public ClassificationTextBlock union(ClassificationTextBlock r) { + public TextPageBlock union(TextPageBlock r) { - ClassificationTextBlock union = this.copy(); + TextPageBlock union = this.copy(); union.add(r); return union; } - public void add(ClassificationTextBlock r) { + public void add(TextPageBlock r) { if (r.getMinX() < minX) { minX = r.getMinX(); @@ -239,9 +236,9 @@ public class ClassificationTextBlock extends AbstractPageBlock { } - public ClassificationTextBlock copy() { + public TextPageBlock copy() { - return new ClassificationTextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage); + return new TextPageBlock(minX, maxX, minY, maxY, sequences, rotation); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java index 6eea5674..50925713 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java @@ -9,6 +9,6 @@ import lombok.Data; @AllArgsConstructor public class UnclassifiedText { - private List textBlocks; + private List textBlocks; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java index 94b83c80..d67447a9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java @@ -14,8 +14,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Orientation; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.RulingTextDirAdjustUtil; @@ -64,7 +64,7 @@ public class BlockificationService { prevOrientation = chunkBlockList.get(chunkBlockList.size() - 1).getOrientation(); } - ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage); + TextPageBlock cb1 = buildTextBlock(chunkWords, indexOnPage); indexOnPage++; chunkBlockList.add(cb1); @@ -106,17 +106,17 @@ public class BlockificationService { } } - ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage); + TextPageBlock cb1 = buildTextBlock(chunkWords, indexOnPage); if (cb1 != null) { chunkBlockList.add(cb1); } Iterator itty = chunkBlockList.iterator(); - ClassificationTextBlock previousLeft = null; - ClassificationTextBlock previousRight = null; + TextPageBlock previousLeft = null; + TextPageBlock previousRight = null; while (itty.hasNext()) { - ClassificationTextBlock block = (ClassificationTextBlock) itty.next(); + TextPageBlock block = (TextPageBlock) itty.next(); if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) { if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) { @@ -142,9 +142,9 @@ public class BlockificationService { } itty = chunkBlockList.iterator(); - ClassificationTextBlock previous = null; + TextPageBlock previous = null; while (itty.hasNext()) { - ClassificationTextBlock block = (ClassificationTextBlock) itty.next(); + TextPageBlock block = (TextPageBlock) itty.next(); if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation() @@ -167,9 +167,9 @@ public class BlockificationService { } - private ClassificationTextBlock buildTextBlock(List wordBlockList, int indexOnPage) { + private TextPageBlock buildTextBlock(List wordBlockList, int indexOnPage) { - ClassificationTextBlock textBlock = null; + TextPageBlock textBlock = null; FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter(); FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter(); @@ -186,15 +186,14 @@ public class BlockificationService { styleFrequencyCounter.add(wordBlock.getFontStyle()); if (textBlock == null) { - textBlock = new ClassificationTextBlock(wordBlock.getMinXDirAdj(), + textBlock = new TextPageBlock(wordBlock.getMinXDirAdj(), wordBlock.getMaxXDirAdj(), wordBlock.getMinYDirAdj(), wordBlock.getMaxYDirAdj(), wordBlockList, - wordBlock.getRotation(), - indexOnPage); + wordBlock.getRotation()); } else { - ClassificationTextBlock spatialEntity = textBlock.union(wordBlock); + TextPageBlock spatialEntity = textBlock.union(wordBlock); textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java index 562ecd94..c695c3c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java @@ -11,7 +11,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils; @Service @@ -80,8 +80,8 @@ public class BodyTextFrameService { for (AbstractPageBlock container : page.getTextBlocks()) { - if (container instanceof ClassificationTextBlock) { - ClassificationTextBlock textBlock = (ClassificationTextBlock) container; + if (container instanceof TextPageBlock) { + TextPageBlock textBlock = (TextPageBlock) container; if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) { continue; } @@ -105,7 +105,7 @@ public class BodyTextFrameService { if (cell == null || cell.getTextBlocks() == null) { continue; } - for (ClassificationTextBlock textBlock : cell.getTextBlocks()) { + for (TextPageBlock textBlock : cell.getTextBlocks()) { expandRectangle(textBlock, page, expansionsRectangle); } } @@ -120,7 +120,7 @@ public class BodyTextFrameService { } - private void expandRectangle(ClassificationTextBlock textBlock, ClassificationPage page, BodyTextFrameExpansionsRectangle expansionsRectangle) { + private void expandRectangle(TextPageBlock textBlock, ClassificationPage page, BodyTextFrameExpansionsRectangle expansionsRectangle) { if (page.getPageWidth() > page.getPageHeight() && page.getRotation() != 0) { if (textBlock.getPdfMinY() < expansionsRectangle.minX) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java index af75d42f..ea325637 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java @@ -10,7 +10,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationDocument; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.PageBlockType; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils; import lombok.RequiredArgsConstructor; @@ -42,14 +42,14 @@ public class ClassificationService { public void classifyPage(ClassificationPage page, ClassificationDocument document, List headlineFontSizes) { for (AbstractPageBlock textBlock : page.getTextBlocks()) { - if (textBlock instanceof ClassificationTextBlock) { - classifyBlock((ClassificationTextBlock) textBlock, page, document, headlineFontSizes); + if (textBlock instanceof TextPageBlock) { + classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes); } } } - public void classifyBlock(ClassificationTextBlock textBlock, ClassificationPage page, ClassificationDocument document, List headlineFontSizes) { + public void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List headlineFontSizes) { var bodyTextFrame = page.getBodyTextFrame(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java index 52d7cefb..000c1343 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java @@ -24,7 +24,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing.PDFLinesTextStripper; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.FileUtils; @@ -157,11 +157,11 @@ public class PdfSegmentationService { // Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame. for (AbstractPageBlock textBlock : page.getTextBlocks()) { - if (textBlock instanceof ClassificationTextBlock) { - if (((ClassificationTextBlock) textBlock).getSequences() == null) { + if (textBlock instanceof TextPageBlock) { + if (((TextPageBlock) textBlock).getSequences() == null) { continue; } - for (TextPositionSequence word : ((ClassificationTextBlock) textBlock).getSequences()) { + for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) { page.getTextHeightCounter().add(word.getTextHeight()); page.getFontCounter().add(word.getFont()); page.getFontSizeCounter().add(word.getFontSize()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java index a6c79da6..48b7ccbf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java @@ -20,7 +20,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText; import lombok.extern.slf4j.Slf4j; @@ -42,9 +42,9 @@ public class SectionsBuilderService { String lastHeadline = ""; TablePageBlock previousTable = null; for (ClassificationPage page : document.getPages()) { - List header = new ArrayList<>(); - List footer = new ArrayList<>(); - List unclassifiedText = new ArrayList<>(); + List header = new ArrayList<>(); + List footer = new ArrayList<>(); + List unclassifiedText = new ArrayList<>(); for (AbstractPageBlock current : page.getTextBlocks()) { if (current.getClassification() == null) { @@ -54,17 +54,17 @@ public class SectionsBuilderService { current.setPage(page.getPageNumber()); if (current.getClassification().equals(PageBlockType.HEADER)) { - header.add((ClassificationTextBlock) current); + header.add((TextPageBlock) current); continue; } if (current.getClassification().equals(PageBlockType.FOOTER)) { - footer.add((ClassificationTextBlock) current); + footer.add((TextPageBlock) current); continue; } if (current.getClassification().equals(PageBlockType.OTHER)) { - unclassifiedText.add((ClassificationTextBlock) current); + unclassifiedText.add((TextPageBlock) current); continue; } @@ -258,7 +258,7 @@ public class SectionsBuilderService { continue; } - ClassificationTextBlock wordBlock = (ClassificationTextBlock) container; + TextPageBlock wordBlock = (TextPageBlock) container; section.getPageBlocks().add(wordBlock); } return section; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java index 9d520e90..7ab5e4dc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java @@ -20,7 +20,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Rectangle; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils; @Service @@ -83,10 +83,10 @@ public class TableExtractionService { List cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical()); - List toBeRemoved = new ArrayList<>(); + List toBeRemoved = new ArrayList<>(); for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) { - ClassificationTextBlock textBlock = (ClassificationTextBlock) abstractPageBlock; + TextPageBlock textBlock = (TextPageBlock) abstractPageBlock; for (Cell cell : cells) { if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), @@ -122,7 +122,7 @@ public class TableExtractionService { Iterator itty = page.getTextBlocks().iterator(); while (itty.hasNext()) { AbstractPageBlock textBlock = itty.next(); - if (textBlock instanceof ClassificationTextBlock ? table.containsBlock((ClassificationTextBlock) textBlock) : table.contains(textBlock) && position == -1) { + if (textBlock instanceof TextPageBlock ? table.containsBlock((TextPageBlock) textBlock) : table.contains(textBlock) && position == -1) { position = page.getTextBlocks().indexOf(textBlock); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java index 63ccfa01..028b7e6c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java @@ -1,7 +1,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import lombok.experimental.UtilityClass; @@ -11,7 +11,7 @@ public final class PositionUtils { // TODO This currently uses pdf coord system. In the futher this should use java coord system. // Note: DirAdj (TextDirection Adjusted) can not be user for this. - public boolean isWithinBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock) { + public boolean isWithinBodyTextFrame(Rectangle btf, TextPageBlock textBlock) { if (btf == null || textBlock == null) { return false; @@ -32,7 +32,7 @@ public final class PositionUtils { // TODO This currently uses pdf coord system. In the futher this should use java coord system. // Note: DirAdj (TextDirection Adjusted) can not be user for this. - public boolean isOverBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock, int rotation) { + public boolean isOverBodyTextFrame(Rectangle btf, TextPageBlock textBlock, int rotation) { if (btf == null || textBlock == null) { return false; @@ -61,7 +61,7 @@ public final class PositionUtils { // TODO This currently uses pdf coord system. In the futher this should use java coord system. // Note: DirAdj (TextDirection Adjusted) can not be user for this. - public boolean isUnderBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock, int rotation) { + public boolean isUnderBodyTextFrame(Rectangle btf, TextPageBlock textBlock, int rotation) { if (btf == null || textBlock == null) { return false; @@ -90,7 +90,7 @@ public final class PositionUtils { // TODO This currently uses pdf coord system. In the futher this should use java coord system. // Note: DirAdj (TextDirection Adjusted) can not be user for this. - public boolean isTouchingUnderBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock) { + public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextPageBlock textBlock) { //TODO Currently this is not working for rotated pages. @@ -107,13 +107,13 @@ public final class PositionUtils { } - public float getHeightDifferenceBetweenChunkWordAndDocumentWord(ClassificationTextBlock textBlock, Float documentMostPopularWordHeight) { + public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextPageBlock textBlock, Float documentMostPopularWordHeight) { return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight; } - public Float getApproxLineCount(ClassificationTextBlock textBlock) { + public Float getApproxLineCount(TextPageBlock textBlock) { return textBlock.getHeight() / textBlock.getMostPopularWordHeight(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java index bb156830..53823da6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java @@ -10,7 +10,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Do import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; @@ -38,9 +37,9 @@ public class DocumentTreeData { if (tocId.isEmpty()) { return root; } - EntryData entry = root.subEntries.get(tocId.get(0)); + EntryData entry = root.children.get(tocId.get(0)); for (int id : tocId.subList(1, tocId.size())) { - entry = entry.subEntries.get(id); + entry = entry.children.get(id); } return entry; } @@ -48,7 +47,7 @@ public class DocumentTreeData { public Stream streamAllEntries() { - return Stream.concat(Stream.of(root), root.subEntries.stream()).flatMap(DocumentTreeData::flatten); + return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentTreeData::flatten); } @@ -60,7 +59,7 @@ public class DocumentTreeData { private static Stream flatten(EntryData entry) { - return Stream.concat(Stream.of(entry), entry.subEntries.stream().flatMap(DocumentTreeData::flatten)); + return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTreeData::flatten)); } @@ -75,7 +74,7 @@ public class DocumentTreeData { Long[] atomicBlockIds; Long[] pageNumbers; Map properties; - List subEntries; + List children; public static EntryData fromEntry(DocumentTree.Entry entry) { @@ -86,7 +85,6 @@ public class DocumentTreeData { case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode()); case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode()); case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode()); - case SECTION -> PropertiesMapper.buildSectionProperties((Section) entry.getNode()); default -> new HashMap<>(); }; var treeId = entry.getTreeId().stream().mapToInt(Integer::intValue).toArray(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java index 37790108..648e7100 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java @@ -1,8 +1,5 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType.FOOTER; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType.HEADER; - import java.util.Arrays; import java.util.HashSet; import java.util.LinkedList; @@ -39,22 +36,18 @@ public class DocumentGraphMapper { public Document toDocumentGraph(DocumentData documentData) { Document document = new Document(); - Context context = new Context(documentData, - new DocumentTree(document), - new LinkedList<>(), - new LinkedList<>(), - Arrays.stream(documentData.getAtomicTextBlocks()).toList(), - Arrays.stream(documentData.getAtomicPositionBlocks()).toList()); + DocumentTree documentTree = new DocumentTree(document); + Context context = new Context(documentData, documentTree); context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList()); - context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentTreeData().getRoot().getSubEntries(), context)); + context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentTreeData().getRoot().getChildren(), context)); document.setDocumentTree(context.documentTree); document.setPages(new HashSet<>(context.pages)); document.setNumberOfPages(documentData.getPages().length); - document.setTextBlock(document.buildTextBlock()); + document.setTextBlock(document.getTextBlock()); return document; } @@ -68,40 +61,39 @@ public class DocumentGraphMapper { List pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList(); SemanticNode node = switch (entryData.getType()) { - case SECTION -> buildSection(context, entryData.getProperties()); - case PARAGRAPH -> buildParagraph(context, terminal); - case HEADLINE -> buildHeadline(context, terminal); - case HEADER -> buildHeader(context, terminal); - case FOOTER -> buildFooter(context, terminal); + case SECTION -> buildSection(context); + case PARAGRAPH -> buildParagraph(context); + case HEADLINE -> buildHeadline(context); + case HEADER -> buildHeader(context); + case FOOTER -> buildFooter(context); case TABLE -> buildTable(context, entryData.getProperties()); - case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal); + case TABLE_CELL -> buildTableCell(context, entryData.getProperties()); case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers()); default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType()); }; - if (node.isLeaf()) { + if (entryData.getAtomicBlockIds().length > 0) { TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node); node.setLeafTextBlock(textBlock); } List treeId = Arrays.stream(entryData.getTreeId()).boxed().toList(); node.setTreeId(treeId); - if (entryData.getType() == HEADER) { - pages.forEach(page -> page.setHeader((Header) node)); - } else if (entryData.getType() == FOOTER) { - pages.forEach(page -> page.setFooter((Footer) node)); - } else { - pages.forEach(page -> page.getMainBody().add(node)); + switch (entryData.getType()) { + case HEADER -> pages.forEach(page -> page.setHeader((Header) node)); + case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node)); + default -> pages.forEach(page -> page.getMainBody().add(node)); } - newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getSubEntries(), context)).node(node).build()); + + newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build()); } return newEntries; } - private Headline buildHeadline(Context context, boolean terminal) { + private Headline buildHeadline(Context context) { - return Headline.builder().leaf(terminal).documentTree(context.documentTree()).build(); + return Headline.builder().documentTree(context.documentTree).build(); } @@ -117,15 +109,15 @@ public class DocumentGraphMapper { Page page = getPage(pageNumbers[0], context); var builder = Image.builder(); PropertiesMapper.parseImageProperties(properties, builder); - return builder.documentTree(context.documentTree()).page(page).build(); + return builder.documentTree(context.documentTree).page(page).build(); } - private TableCell buildTableCell(Context context, Map properties, boolean terminal) { + private TableCell buildTableCell(Context context, Map properties) { TableCell.TableCellBuilder builder = TableCell.builder(); PropertiesMapper.parseTableCellProperties(properties, builder); - return builder.leaf(terminal).documentTree(context.documentTree()).build(); + return builder.documentTree(context.documentTree).build(); } @@ -133,33 +125,31 @@ public class DocumentGraphMapper { Table.TableBuilder builder = Table.builder(); PropertiesMapper.parseTableProperties(properties, builder); - return builder.documentTree(context.documentTree()).build(); + return builder.documentTree(context.documentTree).build(); } - private Footer buildFooter(Context context, boolean terminal) { + private Footer buildFooter(Context context) { - return Footer.builder().leaf(terminal).documentTree(context.documentTree()).build(); + return Footer.builder().documentTree(context.documentTree).build(); } - private Header buildHeader(Context context, boolean terminal) { + private Header buildHeader(Context context) { - return Header.builder().leaf(terminal).documentTree(context.documentTree()).build(); + return Header.builder().documentTree(context.documentTree).build(); } - private Section buildSection(Context context, Map properties) { + private Section buildSection(Context context) { - var builder = Section.builder(); - PropertiesMapper.parseSectionProperties(properties, builder); - return builder.documentTree(context.documentTree()).build(); + return Section.builder().documentTree(context.documentTree).build(); } - private Paragraph buildParagraph(Context context, boolean terminal) { + private Paragraph buildParagraph(Context context) { - return Paragraph.builder().leaf(terminal).documentTree(context.documentTree()).build(); + return Paragraph.builder().documentTree(context.documentTree).build(); } @@ -193,13 +183,22 @@ public class DocumentGraphMapper { } - record Context( - DocumentData layoutParsingModel, - DocumentTree documentTree, - List pages, - List
sections, - List atomicTextBlockData, - List atomicPositionBlockData) { + static final class Context { + + private final DocumentTree documentTree; + private final List pages; + private final List atomicTextBlockData; + private final List atomicPositionBlockData; + + + Context(DocumentData documentData, DocumentTree documentTree) { + + this.documentTree = documentTree; + this.pages = new LinkedList<>(); + this.atomicTextBlockData = Arrays.stream(documentData.getAtomicTextBlocks()).toList(); + this.atomicPositionBlockData = Arrays.stream(documentData.getAtomicPositionBlocks()).toList(); + + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java index 6c1fb9ef..eaebbe38 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java @@ -1,42 +1,61 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper.parseRectangle2D; +import static java.lang.String.format; +import java.awt.geom.Rectangle2D; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper; public class PropertiesMapper { + private static final String imageType = "imageType"; + private static final String transparency = "transparency"; + private static final String position = "position"; + private static final String id = "id"; + private static final String row = "row"; + private static final String col = "col"; + private static final String header = "header"; + private static final String bBox = "bBox"; + private static final String numberOfRows = "numberOfRows"; + private static final String numberOfCols = "numberOfCols"; + + public static Map buildImageProperties(Image image) { Map properties = new HashMap<>(); - properties.put("imageType", image.getImageType().toString()); - properties.put("transparency", String.valueOf(image.isTransparent())); - properties.put("position", RectangleMapper.toString(image.getPosition())); - properties.put("id", image.getId()); + properties.put(imageType, image.getImageType().toString()); + properties.put(transparency, String.valueOf(image.isTransparent())); + properties.put(position, toString(image.getPosition())); + properties.put(id, image.getId()); return properties; } + private static String toString(Rectangle2D rectangle2D) { + + return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight()); + } + + public static Map buildTableCellProperties(TableCell tableCell) { Map properties = new HashMap<>(); - properties.put("row", String.valueOf(tableCell.getRow())); - properties.put("col", String.valueOf(tableCell.getCol())); - properties.put("header", String.valueOf(tableCell.isHeader())); + properties.put(row, String.valueOf(tableCell.getRow())); + properties.put(col, String.valueOf(tableCell.getCol())); + properties.put(header, String.valueOf(tableCell.isHeader())); if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) { throw new IllegalArgumentException("TableCell can only occur on a single page!"); } - String bBoxString = RectangleMapper.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get())); - properties.put("bBox", bBoxString); + String bBoxString = toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get())); + properties.put(bBox, bBoxString); return properties; } @@ -45,60 +64,41 @@ public class PropertiesMapper { public static Map buildTableProperties(Table table) { Map properties = new HashMap<>(); - properties.put("numberOfRows", String.valueOf(table.getNumberOfRows())); - properties.put("numberOfCols", String.valueOf(table.getNumberOfCols())); - return properties; - } - - - public static Map buildSectionProperties(Section node) { - - Map properties = new HashMap<>(); - properties.put("excludesTables", String.valueOf(node.isExcludesTables())); + properties.put(numberOfRows, String.valueOf(table.getNumberOfRows())); + properties.put(numberOfCols, String.valueOf(table.getNumberOfCols())); return properties; } public static void parseImageProperties(Map properties, Image.ImageBuilder builder) { - builder.imageType(parseImageType(properties.get("imageType"))); - builder.transparent(Boolean.parseBoolean(properties.get("transparency"))); - builder.position(parseRectangle2D(properties.get("position"))); - builder.id(properties.get("id")); + builder.imageType(ImageType.fromString(properties.get(imageType))); + builder.transparent(Boolean.parseBoolean(properties.get(transparency))); + builder.position(parseRectangle2D(properties.get(position))); + builder.id(properties.get(id)); } public static void parseTableCellProperties(Map properties, TableCell.TableCellBuilder builder) { - builder.row(Integer.parseInt(properties.get("row"))); - builder.col(Integer.parseInt(properties.get("col"))); - builder.header(Boolean.parseBoolean(properties.get("header"))); - builder.bBox(parseRectangle2D(properties.get("bBox"))); + builder.row(Integer.parseInt(properties.get(row))); + builder.col(Integer.parseInt(properties.get(col))); + builder.header(Boolean.parseBoolean(properties.get(header))); + builder.bBox(parseRectangle2D(properties.get(bBox))); } public static void parseTableProperties(Map properties, Table.TableBuilder builder) { - builder.numberOfRows(Integer.parseInt(properties.get("numberOfRows"))); - builder.numberOfCols(Integer.parseInt(properties.get("numberOfCols"))); + builder.numberOfRows(Integer.parseInt(properties.get(numberOfRows))); + builder.numberOfCols(Integer.parseInt(properties.get(numberOfCols))); } - public static void parseSectionProperties(Map properties, Section.SectionBuilder builder) { + private static Rectangle2D parseRectangle2D(String bBox) { - builder.excludesTables(Boolean.parseBoolean(properties.get("excludesTables"))); - } - - - public static ImageType parseImageType(String imageType) { - - return switch (imageType.toLowerCase()) { - case "logo" -> ImageType.LOGO; - case "formula" -> ImageType.FORMULA; - case "signature" -> ImageType.SIGNATURE; - case "ocr" -> ImageType.OCR; - default -> ImageType.OTHER; - }; + List floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList(); + return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3)); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java index b6a5fc63..d42c4216 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java @@ -5,6 +5,7 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import java.awt.geom.Rectangle2D; +import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -18,7 +19,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationHeader; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Footer; @@ -56,7 +57,7 @@ public class DocumentGraphFactory { documentGraph.setNumberOfPages(context.pages.size()); documentGraph.setPages(context.pages.keySet()); documentGraph.setDocumentTree(context.documentTree); - documentGraph.setTextBlock(documentGraph.buildTextBlock()); + documentGraph.setTextBlock(documentGraph.getTextBlock()); return documentGraph; } @@ -67,10 +68,7 @@ public class DocumentGraphFactory { } - public void addParagraphOrHeadline(GenericSemanticNode parentNode, - ClassificationTextBlock originalTextBlock, - Context context, - List textBlocksToMerge) { + public void addParagraphOrHeadline(GenericSemanticNode parentNode, TextPageBlock originalTextBlock, Context context, List textBlocksToMerge) { Page page = context.getPage(originalTextBlock.getPage()); @@ -83,7 +81,7 @@ public class DocumentGraphFactory { page.getMainBody().add(node); - List textBlocks = new LinkedList<>(textBlocksToMerge); + List textBlocks = new ArrayList<>(textBlocksToMerge); textBlocks.add(originalTextBlock); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page); List treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node); @@ -113,13 +111,13 @@ public class DocumentGraphFactory { private void addHeaderAndFooterToEachPage(ClassificationDocument document, Context context) { - Map> headers = document.getHeaders() + Map> headers = document.getHeaders() .stream() .map(ClassificationHeader::getTextBlocks) .flatMap(List::stream) .collect(groupingBy(AbstractPageBlock::getPage, toList())); - Map> footers = document.getFooters() + Map> footers = document.getFooters() .stream() .map(ClassificationFooter::getTextBlocks) .flatMap(List::stream) @@ -143,7 +141,7 @@ public class DocumentGraphFactory { } - private void addFooter(List textBlocks, Context context) { + private void addFooter(List textBlocks, Context context) { Page page = context.getPage(textBlocks.get(0).getPage()); Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build(); @@ -158,7 +156,7 @@ public class DocumentGraphFactory { } - public void addHeader(List textBlocks, Context context) { + public void addHeader(List textBlocks, Context context) { Page page = context.getPage(textBlocks.get(0).getPage()); Header header = Header.builder().documentTree(context.getDocumentTree()).build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java index 902b9d36..3a2601bd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java @@ -13,6 +13,12 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo public class SearchTextWithTextPositionFactory { public static final int HEIGHT_PADDING = 2; + // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away. + // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height. + // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate + // This is why, we need to initialize this to < -2, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there. + // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3. + public static final int MAX_HYPHEN_LINEBREAK_DISTANCE = 3; public static SearchTextWithTextPositionDto buildSearchTextToTextPositionModel(List sequences) { @@ -30,8 +36,10 @@ public class SearchTextWithTextPositionFactory { for (int i = 0; i < word.getTextPositions().size(); ++i) { currentTextPosition = word.getTextPositions().get(i); - - removeHyphenLinebreaks(currentTextPosition, previousTextPosition, context); + if (isLineBreak(currentTextPosition, previousTextPosition)) { + removeHyphenLinebreaks(context); + context.lineBreaksStringIdx.add(context.stringIdx); + } if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) { if (isHyphen(currentTextPosition.getUnicode())) { context.lastHyphenIdx = context.stringIdx; @@ -76,24 +84,20 @@ public class SearchTextWithTextPositionFactory { } - private static void removeHyphenLinebreaks(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition, Context context) { + private static void removeHyphenLinebreaks(Context context) { - if (isLineBreak(currentTextPosition, previousTextPosition)) { - - if (lastHyphenDirectlyBeforeLineBreak(context)) { - context.stringBuilder.delete(context.lastHyphenIdx, context.stringBuilder.length()); - context.stringIdxToPositionIdx = context.stringIdxToPositionIdx.subList(0, context.lastHyphenIdx); - context.stringIdx = context.lastHyphenIdx; - context.lastHyphenIdx = -3; - } - context.lineBreaksStringIdx.add(context.stringIdx); + if (lastHyphenDirectlyBeforeLineBreak(context)) { + context.stringBuilder.delete(context.lastHyphenIdx, context.stringBuilder.length()); + context.stringIdxToPositionIdx = context.stringIdxToPositionIdx.subList(0, context.lastHyphenIdx); + context.stringIdx = context.lastHyphenIdx; + context.lastHyphenIdx = -MAX_HYPHEN_LINEBREAK_DISTANCE; } } private static boolean lastHyphenDirectlyBeforeLineBreak(Context context) { - return context.stringIdx - context.lastHyphenIdx < 3; + return context.stringIdx - context.lastHyphenIdx < MAX_HYPHEN_LINEBREAK_DISTANCE; } @@ -170,12 +174,8 @@ public class SearchTextWithTextPositionFactory { int stringIdx; int positionIdx; - // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away. - // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height. - // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate - // This is why, we need to initialize this to < -2, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there. - // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3. - int lastHyphenIdx = -3; + + int lastHyphenIdx = -MAX_HYPHEN_LINEBREAK_DISTANCE; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java index 37374f5e..6a97e731 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory; +import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.groupingBy; @@ -12,7 +13,7 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.GenericSemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; @@ -29,7 +30,7 @@ public class SectionNodeFactory { return; } Map> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage)); - Section section = Section.builder().entities(new HashSet<>()).documentTree(context.getDocumentTree()).build(); + Section section = Section.builder().documentTree(context.getDocumentTree()).build(); context.getSections().add(section); blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber)); @@ -41,9 +42,6 @@ public class SectionNodeFactory { splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(section, subSectionPageBlocks, emptyList(), context)); } else { addTablesAndParagraphsAndHeadlinesToSection(pageBlocks, context, section); - if (listIsTextBlockOnly(pageBlocks)) { - section.setExcludesTables(true); - } } images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(section, image, context)); @@ -60,12 +58,6 @@ public class SectionNodeFactory { } - private boolean listIsTextBlockOnly(List pageBlocks) { - - return pageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof ClassificationTextBlock); - } - - private void addFirstHeadlineDirectlyToSection(List pageBlocks, DocumentGraphFactory.Context context, Section section) { if (pageBlocks.get(0).isHeadline()) { @@ -87,16 +79,16 @@ public class SectionNodeFactory { remainingBlocks.removeAll(alreadyMerged); - if (abstractPageBlock instanceof ClassificationTextBlock) { - List textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, remainingBlocks); + if (abstractPageBlock instanceof TextPageBlock) { + List textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, remainingBlocks); alreadyMerged.addAll(textBlocks); - DocumentGraphFactory.addParagraphOrHeadline(section, (ClassificationTextBlock) abstractPageBlock, context, textBlocks); - } - if (abstractPageBlock instanceof TablePageBlock tablePageBlock) { - + DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks); + } else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) { List tablesToMerge = TableMergingUtility.findConsecutiveTablesWithSameColCountAndSameHeaders(tablePageBlock, remainingBlocks); alreadyMerged.addAll(tablesToMerge); TableNodeFactory.addTable(section, tablesToMerge, context); + } else { + throw new RuntimeException(format("Unhandled AbstractPageBlockType %s!", abstractPageBlock.getClass())); } } } @@ -104,8 +96,7 @@ public class SectionNodeFactory { private boolean containsTablesAndTextBlocks(List pageBlocks) { - return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream() - .anyMatch(pageBlock -> pageBlock instanceof ClassificationTextBlock); + return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TextPageBlock); } @@ -171,14 +162,14 @@ public class SectionNodeFactory { } - private List findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List pageBlocks) { + private List findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List pageBlocks) { return pageBlocks.stream() .filter(abstractTextContainer -> !abstractTextContainer.equals(atc)) .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage()) - .filter(abstractTextContainer -> abstractTextContainer instanceof ClassificationTextBlock) + .filter(abstractTextContainer -> abstractTextContainer instanceof TextPageBlock) .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc)) - .map(abstractTextContainer -> (ClassificationTextBlock) abstractTextContainer) + .map(abstractTextContainer -> (TextPageBlock) abstractTextContainer) .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java index 1d8b612a..c0e79d78 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java @@ -36,11 +36,11 @@ public class TableNodeFactory { pages.forEach(page -> addTableToPage(page, parentNode, table)); - List tocId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table); - table.setTreeId(tocId); + List treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table); + table.setTreeId(treeId); addTableCells(mergedRows, table, context); - ifTableHasNoHeadersAssumeFirstRowAreHeaders(table); + ifTableHasNoHeadersSetFirstRowAsHeaders(table); } @@ -75,7 +75,7 @@ public class TableNodeFactory { } - private void ifTableHasNoHeadersAssumeFirstRowAreHeaders(Table table) { + private void ifTableHasNoHeadersSetFirstRowAsHeaders(Table table) { if (table.streamHeaders().findAny().isEmpty()) { table.streamRow(0).forEach(tableCellNode -> tableCellNode.setHeader(true)); @@ -101,35 +101,24 @@ public class TableNodeFactory { TableCell tableCell = TableCell.builder().documentTree(context.getDocumentTree()).row(rowIndex).col(colIndex).header(cell.isHeaderCell()).bBox(cell.getBounds2D()).build(); page.getMainBody().add(tableCell); + List treeId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell); + tableCell.setTreeId(treeId); + TextBlock textBlock; - - List tocId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell); - tableCell.setTreeId(tocId); - if (cell.getTextBlocks().isEmpty()) { tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page)); - tableCell.setLeaf(true); - } else if (cell.getTextBlocks().size() == 1) { textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page); tableCell.setLeafTextBlock(textBlock); - tableCell.setLeaf(true); - } else if (firstTextBlockIsHeadline(cell)) { SectionNodeFactory.addSection(tableCell, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context); - tableCell.setLeaf(false); - } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) { List sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks()); textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page); tableCell.setLeafTextBlock(textBlock); - tableCell.setLeaf(true); - } else { cell.getTextBlocks().forEach(tb -> DocumentGraphFactory.addParagraphOrHeadline(tableCell, tb, context, emptyList())); - tableCell.setLeaf(false); } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java index 488f1f50..e581082c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java @@ -6,9 +6,13 @@ import java.util.Collection; import java.util.LinkedList; import java.util.List; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; + +import lombok.EqualsAndHashCode; import lombok.Setter; @Setter +@EqualsAndHashCode public class Boundary implements Comparable { private int start; @@ -135,17 +139,25 @@ public class Boundary implements Comparable { } - @Override - public int hashCode() { + /** + * shrinks the boundary, such that textBlock.subSequence(boundary) returns a string without whitespaces. + * + * @param textBlock TextBlock to check whitespaces against + * @return boundary + */ + public Boundary trim(TextBlock textBlock) { - return toString().hashCode(); - } + int trimmedStart = this.start; + while (Character.isWhitespace(textBlock.charAt(trimmedStart))) { + trimmedStart++; + } + int trimmedEnd = this.end; + while (Character.isWhitespace(textBlock.charAt(trimmedEnd - 1))) { + trimmedEnd--; + } - @Override - public boolean equals(Object object) { - - return hashCode() == object.hashCode(); + return new Boundary(trimmedStart, Math.max(trimmedEnd, trimmedStart)); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java index 6ca32f5e..674a4ded 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java @@ -2,13 +2,11 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph; import static java.lang.String.format; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.stream.Stream; -import com.google.common.hash.Hashing; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.GenericSemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType; @@ -22,10 +20,12 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.experimental.FieldDefaults; @Data +@EqualsAndHashCode public class DocumentTree { private final Entry root; @@ -39,7 +39,7 @@ public class DocumentTree { public TextBlock buildTextBlock() { - return streamAllEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + return allEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); } @@ -71,13 +71,13 @@ public class DocumentTree { private List createNewChildEntryAndReturnIdImpl(List parentId, SemanticNode node) { if (!entryExists(parentId)) { - throw new UnsupportedOperationException(format("parentId %s does not exist!", parentId)); + throw new IllegalArgumentException(format("parentId %s does not exist!", parentId)); } Entry parent = getEntryById(parentId); List newId = new LinkedList<>(parentId); newId.add(parent.children.size()); - parent.children.add(Entry.builder().treeId(newId).node(node).children(new LinkedList<>()).build()); + parent.children.add(Entry.builder().treeId(newId).node(node).build()); return newId; } @@ -111,13 +111,13 @@ public class DocumentTree { } - public Stream streamChildNodes(List treeId) { + public Stream childNodes(List treeId) { return getEntryById(treeId).children.stream().map(Entry::getNode); } - public Stream streamChildNodesOfType(List treeId, NodeType nodeType) { + public Stream childNodesOfType(List treeId, NodeType nodeType) { return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode); } @@ -148,19 +148,19 @@ public class DocumentTree { } - public Stream streamMainEntries() { + public Stream mainEntries() { return root.children.stream(); } - public Stream streamAllEntriesInOrder() { + public Stream allEntriesInOrder() { return Stream.of(root).flatMap(DocumentTree::flatten); } - public Stream streamAllSubEntriesInOrder(List parentId) { + public Stream allSubEntriesInOrder(List parentId) { return getEntryById(parentId).children.stream().flatMap(DocumentTree::flatten); } @@ -169,7 +169,7 @@ public class DocumentTree { @Override public String toString() { - return String.join("\n", streamAllEntriesInOrder().map(Entry::toString).toList()); + return String.join("\n", allEntriesInOrder().map(Entry::toString).toList()); } @@ -196,7 +196,8 @@ public class DocumentTree { List treeId; SemanticNode node; - List children; + @Builder.Default + List children = new LinkedList<>(); @Override @@ -211,20 +212,6 @@ public class DocumentTree { return node.getType(); } - - @Override - public int hashCode() { - - return Hashing.murmur3_32_fixed().hashString(toString(), StandardCharsets.UTF_8).hashCode(); - } - - - @Override - public boolean equals(Object o) { - - return o instanceof Entry && o.hashCode() == this.hashCode(); - } - } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java index 0f3b9f04..fb6ee428 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java @@ -1,7 +1,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity; import java.awt.geom.Rectangle2D; -import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Comparator; import java.util.Deque; @@ -11,7 +10,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.hash.Hashing; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; @@ -22,17 +20,22 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.experimental.FieldDefaults; @Data @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class RedactionEntity { // initial values + @EqualsAndHashCode.Include final Boundary boundary; + @EqualsAndHashCode.Include final String type; + @EqualsAndHashCode.Include final EntityType entityType; // empty defaults @@ -51,6 +54,7 @@ public class RedactionEntity { String legalBasis; // inferred on graph insertion + @EqualsAndHashCode.Include String value; String textBefore; String textAfter; @@ -64,22 +68,7 @@ public class RedactionEntity { public static RedactionEntity initialEntityNode(Boundary boundary, String type, EntityType entityType) { - return RedactionEntity.builder() - .type(type) - .entityType(entityType) - .boundary(boundary) - .redaction(false) - .removed(false) - .ignored(false) - .resized(false) - .skipRemoveEntitiesContainedInLarger(false) - .dictionaryEntry(false) - .dossierDictionaryEntry(false) - .engines(new HashSet<>()) - .references(new HashSet<>()) - .redactionReason("") - .legalBasis("") - .build(); + return RedactionEntity.builder().type(type).entityType(entityType).boundary(boundary).engines(new HashSet<>()).references(new HashSet<>()).build(); } @@ -134,7 +123,7 @@ public class RedactionEntity { public int getMatchedRule() { if (matchedRules.isEmpty()) { - return -1; + return 0; } return matchedRules.getLast(); } @@ -143,7 +132,7 @@ public class RedactionEntity { public List getRedactionPositionsPerPage() { if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) { - Map> rectanglesPerLinePerPage = deepestFullyContainingNode.buildTextBlock().getPositionsPerPage(boundary); + Map> rectanglesPerLinePerPage = deepestFullyContainingNode.getTextBlock().getPositionsPerPage(boundary); Page firstPage = rectanglesPerLinePerPage.keySet() .stream() @@ -237,18 +226,4 @@ public class RedactionEntity { return sb.toString(); } - - @Override - public int hashCode() { - - return Hashing.murmur3_128().hashString(toString(), StandardCharsets.UTF_8).hashCode(); - } - - - @Override - public boolean equals(Object o) { - - return o.hashCode() == hashCode(); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionPosition.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionPosition.java index 211d0d6b..73981a1f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionPosition.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionPosition.java @@ -17,6 +17,8 @@ public class RedactionPosition { final String id; Page page; + // Each entry in this list corresponds to an entry in the redaction log, this means: + // An entity might be represented by multiple redaction log entries List rectanglePerLine; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java index 53c4021e..08cba8f7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java @@ -45,9 +45,12 @@ public class Document implements GenericSemanticNode { } - public TextBlock buildTextBlock() { + public TextBlock getTextBlock() { - return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector()); + if (textBlock == null) { + textBlock = streamTerminalTextBlocksInOrder().collect(new TextBlockCollector()); + } + return textBlock; } @@ -86,7 +89,7 @@ public class Document implements GenericSemanticNode { private Stream streamAllNodes() { - return documentTree.streamAllEntriesInOrder().map(DocumentTree.Entry::getNode); + return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode); } @@ -99,7 +102,7 @@ public class Document implements GenericSemanticNode { @Override public String toString() { - return NodeType.DOCUMENT + ": " + buildTextBlock().buildSummary(); + return NodeType.DOCUMENT + ": " + this.getTextBlock().buildSummary(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java index d5fe0e2d..42688e85 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java @@ -26,9 +26,6 @@ public class Footer implements GenericSemanticNode { List treeId; TextBlock leafTextBlock; - @Builder.Default - boolean leaf = true; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @@ -45,7 +42,14 @@ public class Footer implements GenericSemanticNode { @Override - public TextBlock buildTextBlock() { + public boolean isLeaf() { + + return true; + } + + + @Override + public TextBlock getTextBlock() { return leafTextBlock; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java index f6a0060e..4b7c50d8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java @@ -26,9 +26,6 @@ public class Header implements GenericSemanticNode { List treeId; TextBlock leafTextBlock; - @Builder.Default - boolean leaf = true; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @@ -37,6 +34,13 @@ public class Header implements GenericSemanticNode { Set entities = new HashSet<>(); + @Override + public boolean isLeaf() { + + return true; + } + + @Override public NodeType getType() { @@ -45,7 +49,7 @@ public class Header implements GenericSemanticNode { @Override - public TextBlock buildTextBlock() { + public TextBlock getTextBlock() { return leafTextBlock; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java index 33e51727..9e0bcfde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java @@ -26,9 +26,6 @@ public class Headline implements GenericSemanticNode { List treeId; TextBlock leafTextBlock; - @Builder.Default - final boolean leaf = true; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @@ -45,7 +42,14 @@ public class Headline implements GenericSemanticNode { @Override - public TextBlock buildTextBlock() { + public boolean isLeaf() { + + return true; + } + + + @Override + public TextBlock getTextBlock() { return leafTextBlock; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java index f61662dc..24385379 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java @@ -63,7 +63,7 @@ public class Image implements GenericSemanticNode { @Override - public TextBlock buildTextBlock() { + public TextBlock getTextBlock() { return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/ImageType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/ImageType.java index 187a5fe1..adf2d4ea 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/ImageType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/ImageType.java @@ -5,5 +5,17 @@ public enum ImageType { FORMULA, SIGNATURE, OTHER, - OCR + OCR; + + + public static ImageType fromString(String imageType) { + + return switch (imageType.toLowerCase()) { + case "logo" -> ImageType.LOGO; + case "formula" -> ImageType.FORMULA; + case "signature" -> ImageType.SIGNATURE; + case "ocr" -> ImageType.OCR; + default -> ImageType.OTHER; + }; + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/NodeType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/NodeType.java index 20c20461..78ea7c49 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/NodeType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/NodeType.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes; +import java.util.Locale; + public enum NodeType { DOCUMENT, SECTION, @@ -9,5 +11,11 @@ public enum NodeType { TABLE_CELL, IMAGE, HEADER, - FOOTER + FOOTER; + + + public String toString() { + + return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java index 6824e9a2..01b78550 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java @@ -24,9 +24,6 @@ public class Paragraph implements GenericSemanticNode { List treeId; TextBlock leafTextBlock; - @Builder.Default - boolean leaf = true; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @@ -43,7 +40,14 @@ public class Paragraph implements GenericSemanticNode { @Override - public TextBlock buildTextBlock() { + public boolean isLeaf() { + + return true; + } + + + @Override + public TextBlock getTextBlock() { return leafTextBlock; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java index acd79f5e..e2a0ddae 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java @@ -29,7 +29,6 @@ public class Section implements GenericSemanticNode { TextBlock textBlock; @EqualsAndHashCode.Exclude DocumentTree documentTree; - boolean excludesTables; @Builder.Default @EqualsAndHashCode.Exclude @@ -43,8 +42,14 @@ public class Section implements GenericSemanticNode { } + public boolean hasTables() { + + return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent(); + } + + @Override - public TextBlock buildTextBlock() { + public TextBlock getTextBlock() { if (textBlock == null) { textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); @@ -56,7 +61,7 @@ public class Section implements GenericSemanticNode { @Override public String toString() { - return treeId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().buildSummary(); + return treeId.toString() + ": " + NodeType.SECTION + ": " + this.getTextBlock().buildSummary(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java index e2f35f2d..6b37627a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java @@ -34,7 +34,7 @@ public interface SemanticNode { * * @return TextBlock containing all AtomicTextBlocks that are located under this Node. */ - TextBlock buildTextBlock(); + TextBlock getTextBlock(); /** @@ -53,7 +53,7 @@ public interface SemanticNode { */ default Set getPages() { - return buildTextBlock().getPages(); + return getTextBlock().getPages(); } @@ -67,7 +67,13 @@ public interface SemanticNode { if (!getBoundary().contains(boundary)) { throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", boundary, getBoundary())); } - return buildTextBlock().getPages(boundary); + return getTextBlock().getPages(boundary); + } + + + default boolean isOnPage(int pageNumber) { + + return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber); } @@ -218,9 +224,9 @@ public interface SemanticNode { */ default Integer getNumberOnPage() { - TextBlock textBlock = buildTextBlock(); + TextBlock textBlock = getTextBlock(); if (!textBlock.getAtomicTextBlocks().isEmpty()) { - return buildTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage(); + return getTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage(); } else { return -1; } @@ -234,7 +240,7 @@ public interface SemanticNode { */ default boolean hasText() { - return !buildTextBlock().isEmpty(); + return !getTextBlock().isEmpty(); } @@ -246,7 +252,7 @@ public interface SemanticNode { */ default boolean containsString(String string) { - return buildTextBlock().getSearchText().contains(string); + return getTextBlock().getSearchText().contains(string); } @@ -270,7 +276,7 @@ public interface SemanticNode { */ default boolean containsStringIgnoreCase(String string) { - return buildTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase()); + return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase()); } @@ -306,7 +312,7 @@ public interface SemanticNode { */ default void addThisToEntityIfIntersects(RedactionEntity redactionEntity) { - TextBlock textBlock = buildTextBlock(); + TextBlock textBlock = getTextBlock(); if (textBlock.getBoundary().intersects(redactionEntity.getBoundary())) { if (textBlock.containsBoundary(redactionEntity.getBoundary())) { redactionEntity.setDeepestFullyContainingNode(this); @@ -326,7 +332,7 @@ public interface SemanticNode { */ default Stream streamChildren() { - return getDocumentTree().streamChildNodes(getTreeId()); + return getDocumentTree().childNodes(getTreeId()); } @@ -337,7 +343,7 @@ public interface SemanticNode { */ default Stream streamChildrenOfType(NodeType nodeType) { - return getDocumentTree().streamChildNodesOfType(getTreeId(), nodeType); + return getDocumentTree().childNodesOfType(getTreeId(), nodeType); } @@ -348,7 +354,7 @@ public interface SemanticNode { */ default Stream streamAllSubNodes() { - return getDocumentTree().streamAllSubEntriesInOrder(getTreeId()).map(DocumentTree.Entry::getNode); + return getDocumentTree().allSubEntriesInOrder(getTreeId()).map(DocumentTree.Entry::getNode); } @@ -359,7 +365,7 @@ public interface SemanticNode { */ default Stream streamAllSubNodesOfType(NodeType nodeType) { - return getDocumentTree().streamAllSubEntriesInOrder(getTreeId()).filter(entry -> entry.getType().equals(nodeType)).map(DocumentTree.Entry::getNode); + return getDocumentTree().allSubEntriesInOrder(getTreeId()).filter(entry -> entry.getType().equals(nodeType)).map(DocumentTree.Entry::getNode); } @@ -370,7 +376,7 @@ public interface SemanticNode { */ default Boundary getBoundary() { - return buildTextBlock().getBoundary(); + return getTextBlock().getBoundary(); } @@ -429,7 +435,7 @@ public interface SemanticNode { */ private Map getBBoxFromLeafTextBlock(Map bBoxPerPage) { - Map> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage)); + Map> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage)); atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs))); return bBoxPerPage; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java index 313beb09..bdbb1ec8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java @@ -66,7 +66,7 @@ public class Table implements SemanticNode { */ public boolean rowContainsStringsIgnoreCase(Integer row, List strings) { - String rowText = streamRow(row).map(TableCell::buildTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(); + String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(); return strings.stream().map(String::toLowerCase).allMatch(rowText::contains); } @@ -95,11 +95,9 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List values) { - List vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList(); - return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream() - .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsAnyString(values))) - .map(TableCell::getEntities) - .flatMap(Collection::stream); + List colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList(); + return streamTableCells().filter(tableCellNode -> colsWithHeader.stream() + .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream); } @@ -158,7 +156,7 @@ public class Table implements SemanticNode { */ public Stream streamTableCellsWithHeader(String header) { - return streamHeaders().filter(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().contains(header)) + return streamHeaders().filter(tableCellNode -> tableCellNode.getTextBlock().getSearchText().contains(header)) .map(TableCell::getCol) .flatMap(this::streamCol) .filter(tableCellNode -> !tableCellNode.isHeader()); @@ -221,7 +219,7 @@ public class Table implements SemanticNode { */ public boolean hasHeader(String header) { - return streamHeaders().anyMatch(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().strip().equals(header)); + return streamHeaders().anyMatch(tableCellNode -> tableCellNode.getTextBlock().getSearchText().strip().equals(header)); } @@ -279,7 +277,7 @@ public class Table implements SemanticNode { @Override - public TextBlock buildTextBlock() { + public TextBlock getTextBlock() { if (textBlock == null) { textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); @@ -291,7 +289,7 @@ public class Table implements SemanticNode { @Override public String toString() { - return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + buildTextBlock().buildSummary(); + return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java index 3d876a85..704ffc16 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java @@ -32,8 +32,6 @@ public class TableCell implements GenericSemanticNode { Rectangle2D bBox; - @Builder.Default - boolean leaf = true; TextBlock leafTextBlock; TextBlock textBlock; @@ -63,9 +61,16 @@ public class TableCell implements GenericSemanticNode { @Override - public TextBlock buildTextBlock() { + public boolean isLeaf() { - if (leaf) { + return getDocumentTree().getEntryById(getTreeId()).getChildren().isEmpty(); + } + + + @Override + public TextBlock getTextBlock() { + + if (isLeaf()) { return leafTextBlock; } @@ -79,7 +84,7 @@ public class TableCell implements GenericSemanticNode { @Override public String toString() { - return treeId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().buildSummary(); + return treeId + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java index 5eb83393..e53fe5c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java @@ -57,6 +57,7 @@ public class ConcatenatedTextBlock implements TextBlock { } this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks()); boundary.setEnd(textBlock.getBoundary().end()); + this.searchText = null; return this; } @@ -160,7 +161,8 @@ public class ConcatenatedTextBlock implements TextBlock { } AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1); - rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, lastTextBlock.getPositionsPerPage(lastTextBlock.getBoundary())); + rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, + lastTextBlock.getPositionsPerPage(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end()))); return rectanglesPerLinePerPage; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index 1436ccda..f7de5d3e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.service import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedEndByRegex; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedStartByRegex; import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; -import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly; import java.util.Collection; import java.util.Collections; @@ -46,7 +45,7 @@ public class EntityCreationService { public Stream betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { - TextBlock textBlock = node.buildTextBlock(); + TextBlock textBlock = node.getTextBlock(); List startBoundaries = RedactionSearchUtility.findBoundariesByString(start, textBlock); List stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, textBlock); @@ -62,22 +61,25 @@ public class EntityCreationService { stopBoundaries.remove(optionalFirstStopBoundary.get()); entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start())); } - return entityBoundaries.stream().filter(boundary -> isValidEntityBoundary(textBlock, boundary)).map(boundary -> byBoundary(boundary, type, entityType, node)); + return entityBoundaries.stream() + .map(boundary -> boundary.trim(textBlock)) + .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) + .map(boundary -> byBoundary(boundary, type, entityType, node)); } public Stream bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) { - return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary()) + return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary()) .stream() - .filter(boundary -> isValidEntityBoundary(node.buildTextBlock(), boundary)) + .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) .map(bounds -> byBoundary(bounds, type, entityType, node)); } public Stream lineAfterStrings(List strings, String type, EntityType entityType, SemanticNode node) { - TextBlock textBlock = node.buildTextBlock(); + TextBlock textBlock = node.getTextBlock(); SearchImplementation searchImplementation = new SearchImplementation(strings, false); return searchImplementation.getBoundaries(textBlock, node.getBoundary()) .stream() @@ -89,7 +91,7 @@ public class EntityCreationService { public Stream lineAfterString(String string, String type, EntityType entityType, SemanticNode node) { - TextBlock textBlock = node.buildTextBlock(); + TextBlock textBlock = node.getTextBlock(); return RedactionSearchUtility.findBoundariesByString(string, textBlock) .stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) @@ -112,13 +114,13 @@ public class EntityCreationService { public Stream byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); + return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); } public Stream byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegexCaseInsensitive(regexPattern, group, node.buildTextBlock()) + return RedactionSearchUtility.findBoundariesByRegexCaseInsensitive(regexPattern, group, node.getTextBlock()) .stream() .map(boundary -> byBoundary(boundary, type, entityType, node)); } @@ -126,13 +128,13 @@ public class EntityCreationService { public Stream byString(String keyword, String type, EntityType entityType, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByString(keyword, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); + return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); } public RedactionEntity bySemanticNode(SemanticNode node, String type, EntityType entityType) { - Boundary boundary = node.buildTextBlock().getBoundary(); + Boundary boundary = node.getTextBlock().getBoundary(); if (boundary.length() > 0) { boundary = new Boundary(boundary.start(), boundary.end() - 1); } @@ -150,7 +152,7 @@ public class EntityCreationService { public RedactionEntity bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) { int expandedEnd = getExpandedEndByRegex(entity, regexPattern); - expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().buildTextBlock()); + expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock()); return byBoundary(new Boundary(entity.getBoundary().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); } @@ -166,7 +168,8 @@ public class EntityCreationService { public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { - RedactionEntity entity = RedactionEntity.initialEntityNode(boundary, type, entityType); + Boundary trimmedBoundary = boundary.trim(node.getTextBlock()); + RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType); addEntityToGraph(entity, node); return entity; } @@ -175,10 +178,13 @@ public class EntityCreationService { public RedactionEntity byEntities(List entitiesToMerge, String type, EntityType entityType, SemanticNode node) { if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) { - throw new IllegalArgumentException("Provided entities can not be merged!"); + throw new IllegalArgumentException("Provided entities can not be merged, since they do not intersect or are not the same type!" + entitiesToMerge); } - if (entitiesToMerge.size() < 2) { - throw new IllegalArgumentException("more than 2 entities are required to merge!"); + if (entitiesToMerge.isEmpty()) { + throw new IllegalArgumentException("No Entities to merge!"); + } + if (entitiesToMerge.size() == 1) { + return entitiesToMerge.get(0); } RedactionEntity mergedEntity = RedactionEntity.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(RedactionEntity::getBoundary).toList()), type, entityType); @@ -224,7 +230,7 @@ public class EntityCreationService { private boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) { - return boundaryIsSurroundedBySeparators(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary); + return boundaryIsSurroundedBySeparators(textBlock, boundary); } @@ -237,7 +243,7 @@ public class EntityCreationService { documentTree.getRoot().getNode().getEntities().add(entity); } catch (NoSuchElementException e) { entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode()); - entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().buildTextBlock()); + entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock()); entity.addIntersectingNode(documentTree.getRoot().getNode()); addToPages(entity); addEntityToNodeEntitySets(entity); @@ -247,19 +253,18 @@ public class EntityCreationService { private void addEntityToGraph(RedactionEntity entity, DocumentTree documentTree) { - SemanticNode containingNode = documentTree.streamChildNodes(Collections.emptyList()) - .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary())) + SemanticNode containingNode = documentTree.childNodes(Collections.emptyList()) + .filter(node -> node.getTextBlock().containsBoundary(entity.getBoundary())) .findFirst() .orElseThrow(() -> new NoSuchElementException("No containing Node found!")); containingNode.addThisToEntityIfIntersects(entity); - TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock(); + TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock(); entityEnrichmentService.enrichEntity(entity, textBlock); addToPages(entity); addEntityToNodeEntitySets(entity); - } @@ -297,7 +302,7 @@ public class EntityCreationService { private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) { - return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end())); + return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end())).trim(textBlock); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java index f5c5382a..7f0a437e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java @@ -34,7 +34,7 @@ public class EntityEnrichmentService { if (!textAfter.isBlank()) { List wordsAfter = splitToWordsAndRemoveEmptyWords(textAfter); int numberOfWordsAfter = Math.min(wordsAfter.size(), redactionServiceSettings.getNumberOfSurroundingWords()); - if (wordsAfter.size() > 0) { + if (!wordsAfter.isEmpty()) { return concatWordsAfter(wordsAfter.subList(0, numberOfWordsAfter), textAfter.startsWith(" ")); } } @@ -49,7 +49,7 @@ public class EntityEnrichmentService { if (!textBefore.isBlank()) { List wordsBefore = splitToWordsAndRemoveEmptyWords(textBefore); int numberOfWordsBefore = Math.min(wordsBefore.size(), redactionServiceSettings.getNumberOfSurroundingWords()); - if (wordsBefore.size() > 0) { + if (!wordsBefore.isEmpty()) { return concatWordsBefore(wordsBefore.subList(wordsBefore.size() - numberOfWordsBefore, wordsBefore.size()), textBefore.endsWith(" ")); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java index 0a6bc1c0..3b919211 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services; +import java.awt.geom.Rectangle2D; import java.util.NoSuchElementException; import org.springframework.stereotype.Service; @@ -8,7 +9,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper; import lombok.RequiredArgsConstructor; @@ -27,7 +27,7 @@ public class ManualRedactionApplicationService { .findFirst() .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!")); - redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(RectangleMapper::toRectangle2D).toList()); + redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualRedactionApplicationService::toRectangle2D).toList()); int newStartOffset; if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) { @@ -46,4 +46,10 @@ public class ManualRedactionApplicationService { entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto); } + + private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) { + + return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight()); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/PdfVisualisationUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/PdfVisualisationUtility.java index 07f2e9f1..838c7c5c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/PdfVisualisationUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/PdfVisualisationUtility.java @@ -33,7 +33,7 @@ public class PdfVisualisationUtility { public void drawDocumentGraph(PDDocument document, Document documentGraph) { - documentGraph.getDocumentTree().streamAllEntriesInOrder().forEach(entry -> drawNode(document, entry)); + documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry)); } @@ -129,7 +129,7 @@ public class PdfVisualisationUtility { rectanglesPerPage.forEach((page, rectangle2D) -> { Rectangle2D paddedRectangle2D = rectangle2D; if (entry.getType() == NodeType.SECTION) { - paddedRectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10); + paddedRectangle2D = pad(rectangle2D, 10, 10); } drawRectangle2DList(document, page.getNumber(), List.of(paddedRectangle2D), options); drawText(buildString(entry), document, new Point2D.Double(paddedRectangle2D.getMinX(), paddedRectangle2D.getMaxY() + 2), page.getNumber(), options); @@ -137,6 +137,12 @@ public class PdfVisualisationUtility { } + private static Rectangle2D pad(Rectangle2D rectangle2D, int deltaX, int deltaY) { + + return new Rectangle2D.Double(rectangle2D.getMinX() - deltaX, rectangle2D.getMinY() - deltaY, rectangle2D.getWidth() + 2 * deltaX, rectangle2D.getHeight() + 2 * deltaY); + } + + private String buildString(DocumentTree.Entry entry) { return entry.getNode().getNumberOnPage() + ": " + entry.getTreeId() + ": " + entry.getType().toString(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleMapper.java deleted file mode 100644 index 43aa842d..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleMapper.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils; - -import static java.lang.String.format; - -import java.awt.geom.Rectangle2D; -import java.util.Arrays; -import java.util.List; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; - -public class RectangleMapper { - //TODO: Move functions into using services - - - public static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) { - - return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(), - redactionLogRectangle.getTopLeft().getY(), - redactionLogRectangle.getWidth(), - redactionLogRectangle.getHeight()); - } - - - public static Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) { - - return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber); - } - - - public static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) { - - return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(), - (float) rectangle2D.getMaxY(), - (float) rectangle2D.getWidth(), - (float) rectangle2D.getHeight(), - pageNumber); - } - - - public static String toString(Rectangle2D rectangle2D) { - - return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight()); - } - - - public static Rectangle2D parseRectangle2D(String bBox) { - - List floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList(); - return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3)); - } - - - public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) { - - return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight()); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java index 1e3ddc2a..9aa2d2f0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java @@ -15,30 +15,12 @@ import java.util.stream.Collector; import org.apache.pdfbox.pdmodel.common.PDRectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; public class RectangleTransformations { - public static Rectangle2D pad(Rectangle2D rectangle2D, int deltaX, int deltaY) { - - return new Rectangle2D.Double(rectangle2D.getMinX() - deltaX, rectangle2D.getMinY() - deltaY, rectangle2D.getWidth() + 2 * deltaX, rectangle2D.getHeight() + 2 * deltaY); - } - - - public static Rectangle2D bBoxUnionAbstractTextContainer(List abstractPageBlocks) { - - return abstractPageBlocks.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion()); - } - - - public static Rectangle2D toRectangle2D(AbstractPageBlock abstractPageBlock) { - - return new Rectangle2D.Float(abstractPageBlock.getMinX(), abstractPageBlock.getMinY(), abstractPageBlock.getWidth(), abstractPageBlock.getHeight()); - } - - public static PDRectangle toPDRectangleUnion(List rectangles) { Rectangle2D rectangle2D = RectangleTransformations.bBoxUnionRectangle(rectangles); @@ -60,7 +42,25 @@ public class RectangleTransformations { public static Rectangle2D bBoxUnionRectangle(List rectangles) { - return rectangles.stream().map(RectangleMapper::toRectangle2D).collect(new Rectangle2DUnion()); + return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion()); + } + + + public static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) { + + return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(), + redactionLogRectangle.getTopLeft().getY() + redactionLogRectangle.getHeight(), + redactionLogRectangle.getWidth(), + -redactionLogRectangle.getHeight()); + } + + + public static Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) { + + return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())), + (float) rectangle2D.getWidth(), + -(float) rectangle2D.getHeight(), + pageNumber); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java index b500cdf8..41721043 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java @@ -1,18 +1,20 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils; +import static java.lang.String.format; + +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; + import lombok.experimental.UtilityClass; -import java.util.LinkedList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static java.lang.String.format; - @UtilityClass public class RedactionSearchUtility { @@ -115,4 +117,11 @@ public class RedactionSearchUtility { return boundaries; } + + public static List findBoundariesByStringIgnoreCase(String searchString, TextBlock textBlock) { + + String searchStringLowerCase = searchString.toLowerCase(Locale.ROOT); + return findBoundariesByString(searchStringLowerCase, textBlock); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java index d524e7d5..04318811 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java @@ -3,12 +3,12 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils; import java.util.Comparator; import java.util.List; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence; public class TextPositionOperations { - public static List mergeAndSortTextPositionSequenceByYThenX(List textBlocks) { + public static List mergeAndSortTextPositionSequenceByYThenX(List textBlocks) { return textBlocks.stream()// .flatMap(tb -> tb.getSequences().stream())// diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java index 0e964a94..8f2a3d5a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java @@ -152,8 +152,8 @@ public class NerEntitiesAdapter { private static Stream addOffsetsAndFlatten(List stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) { - nerEntitiesModel.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> { - int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key); + nerEntitiesModel.getData().forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> { + int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(sectionNumber); entityRecognitionEntity.setStartOffset(newStartOffset); entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length()); })); @@ -163,7 +163,7 @@ public class NerEntitiesAdapter { private static List getStringStartOffsetsForMainSections(Document document) { - return document.getMainSections().stream().map(Section::buildTextBlock).map(TextBlock::getBoundary).map(Boundary::start).toList(); + return document.getMainSections().stream().map(Section::getTextBlock).map(TextBlock::getBoundary).map(Boundary::start).toList(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java index b33c1163..919e7a36 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java @@ -90,6 +90,9 @@ public class Dictionary { public void addLocalDictionaryEntry(String type, String value, boolean alsoAddLastname) { + if (value.isBlank()) { + return; + } if (localAccessMap.get(type) == null) { throw new IllegalArgumentException(format("DictionaryModel of type %s does not exist", type)); } @@ -99,7 +102,7 @@ public class Dictionary { if (StringUtils.isEmpty(value)) { throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value)); } - localAccessMap.get(type).getLocalEntries().add(value); + localAccessMap.get(type).getLocalEntries().add(value.trim()); if (alsoAddLastname) { String lastname = value.split(" ")[0]; localAccessMap.get(type).getLocalEntries().add(lastname); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 92931e7b..e43ed1b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -345,7 +345,7 @@ public class AnalyzeService { private SimplifiedSectionText toSimplifiedSectionText(Section section) { - return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.buildTextBlock().getSearchText()).build(); + return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java index 024af3f6..6b64b38a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java @@ -22,7 +22,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -43,10 +42,10 @@ public class ManualRedactionSurroundingTextService { public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) { long startTime = System.currentTimeMillis(); - Document text = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId)); + Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId)); List processedAddRedactions = new ArrayList<>(); - for (SemanticNode node : text.streamChildren().toList()) { + for (SemanticNode node : document.streamChildren().toList()) { if (manualRedactions.getEntriesToAdd().isEmpty()) { break; @@ -74,7 +73,7 @@ public class ManualRedactionSurroundingTextService { private Pair findSurroundingText(SemanticNode node, String value, List toFindPositions) { - Set entities = RedactionSearchUtility.findBoundariesByString(value, node.buildTextBlock()) + Set entities = RedactionSearchUtility.findBoundariesByString(value, node.getTextBlock()) .stream() .map(boundary -> entityCreationService.byBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node)) .collect(Collectors.toSet()); @@ -87,7 +86,7 @@ public class ManualRedactionSurroundingTextService { private boolean sectionContainsEntry(SemanticNode semanticNode, List positions) { for (Rectangle position : positions) { - if (semanticNode.containsRectangle(RectangleMapper.toRectangle2D(position), position.getPage())) { + if (semanticNode.containsRectangle(ManualRedactionSurroundingTextService.toRectangle2D(position), position.getPage())) { return true; } } @@ -110,7 +109,13 @@ public class ManualRedactionSurroundingTextService { private static boolean toFindPositionsIntersectRectangle(List toFindPositions, Rectangle2D rectangle2D) { - return toFindPositions.stream().map(RectangleMapper::toRectangle2D).anyMatch(toFindRectangle -> toFindRectangle.intersects(rectangle2D)); + return toFindPositions.stream().map(ManualRedactionSurroundingTextService::toRectangle2D).anyMatch(toFindRectangle -> toFindRectangle.intersects(rectangle2D)); + } + + + private static Rectangle2D toRectangle2D(Rectangle rect) { + + return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 47a0efd4..a1ae8682 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -1,7 +1,5 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper.toRedactionLogRectangle; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -17,6 +15,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -67,7 +66,7 @@ public class RedactionLogCreatorService { List rectanglesPerLine = redactionPosition.getRectanglePerLine() .stream() - .map(rectangle2D -> toRedactionLogRectangle(rectangle2D, redactionPosition.getPage().getNumber())) + .map(rectangle2D -> RectangleTransformations.toRedactionLogRectangle(rectangle2D, redactionPosition.getPage().getNumber())) .toList(); redactionLogEntry.setPositions(rectanglesPerLine); @@ -85,7 +84,7 @@ public class RedactionLogCreatorService { .stream() .filter(redactionEntity -> !redactionEntity.isRemoved() && !redactionEntity.isIgnored()) .forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId()))); - int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? -1 : entity.getDeepestFullyContainingNode().getTreeId().get(0); + int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); return RedactionLogEntry.builder() .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction())) @@ -127,9 +126,9 @@ public class RedactionLogCreatorService { .isHint(dictionaryService.isHint(image.getImageType().toString(), dossierTemplateId)) .isDictionaryEntry(false) .isRecommendation(false) - .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber()))) + .positions(List.of(RectangleTransformations.toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber()))) .sectionNumber(image.getTreeId().get(0)) - .section(image.getParent().buildTextBlock().getSearchText()) + .section(image.getParent().getTextBlock().getSearchText()) .imageHasTransparency(image.isTransparent()) .build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java index 4fa9ea20..12347b42 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java @@ -50,7 +50,7 @@ class SectionFinderService { true); document.streamChildren().forEach(mainNode -> { - if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.buildTextBlock().getSearchText())) { + if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) { sectionsToReanalyse.add(mainNode.getTreeId().get(0)); } }); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java index f1914369..00657234 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java @@ -13,7 +13,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationSection; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock; -import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock; import lombok.RequiredArgsConstructor; @@ -41,9 +41,9 @@ public class SectionGridCreatorService { continue; } - if (textBlock instanceof ClassificationTextBlock) { + if (textBlock instanceof TextPageBlock) { - ClassificationTextBlock tb = (ClassificationTextBlock) textBlock; + TextPageBlock tb = (TextPageBlock) textBlock; classifiedDoc.getSectionGrid() .getRectanglesPerPage() .computeIfAbsent(page, (x) -> new ArrayList<>()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java index 0a297c67..bb9a87d8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java @@ -2,8 +2,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; import java.awt.geom.Rectangle2D; import java.nio.charset.StandardCharsets; +import java.util.Comparator; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; @@ -19,9 +21,19 @@ public final class IdBuilder { public String buildId(Set pages, List rectanglesPerLine) { + return buildId(pages.stream().map(Page::getNumber).collect(Collectors.toList()), rectanglesPerLine); + } + + + public String buildId(List pageNumbers, List rectanglesPerLine) { + StringBuilder sb = new StringBuilder(); - pages.forEach(pageNode -> sb.append(pageNode.getNumber())); - rectanglesPerLine.forEach(rectangle2D -> sb.append(rectangle2D.getX()).append(rectangle2D.getY()).append(rectangle2D.getWidth()).append(rectangle2D.getHeight())); + List sortedPageNumbers = pageNumbers.stream().sorted(Comparator.comparingInt(Integer::intValue)).toList(); + sortedPageNumbers.forEach(sb::append); + rectanglesPerLine.forEach(rectangle2D -> sb.append(Math.round(rectangle2D.getX())) + .append(Math.round(rectangle2D.getY())) + .append(Math.round(rectangle2D.getWidth())) + .append(Math.round(rectangle2D.getHeight()))); return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java index d26859ae..544c44c3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java @@ -47,7 +47,7 @@ public final class SeparatorUtils { public static boolean boundaryIsSurroundedBySeparators(TextBlock textBlock, Boundary boundary) { - return validateStart(textBlock, boundary) && validateEnd(textBlock, boundary); + return validateStart(textBlock, boundary) && validateEnd(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary); } @@ -55,7 +55,7 @@ public final class SeparatorUtils { return boundary.end() == textBlock.getBoundary().end() ||// SeparatorUtils.isSeparator(textBlock.charAt(boundary.end())) ||// - SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1)); + SeparatorUtils.isJapaneseSeparator(textBlock.charAt(boundary.end() - 1)); } @@ -63,7 +63,7 @@ public final class SeparatorUtils { return boundary.start() == textBlock.getBoundary().start() ||// SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) ||// - SeparatorUtils.isSeparator(textBlock.charAt(boundary.start())); + SeparatorUtils.isJapaneseSeparator(textBlock.charAt(boundary.start())); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java index cc486895..42f0337a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java @@ -104,7 +104,7 @@ public class HeadlinesGoldStandardIntegrationTest { var foundHeadlines = documentGraph.streamAllSubNodes() .map(SemanticNode::getHeadline) .distinct() - .map(headlineNode -> new Headline(headlineNode.getPages().stream().findFirst().get().getNumber(), headlineNode.buildTextBlock().getSearchText().stripTrailing())) + .map(headlineNode -> new Headline(headlineNode.getPages().stream().findFirst().get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing())) .toList(); Set correct = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index b0721927..5627dcfa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; import static org.wildfly.common.Assert.assertTrue; +import java.awt.geom.Rectangle2D; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -16,6 +17,7 @@ import java.nio.file.Paths; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -58,6 +60,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; @@ -70,7 +73,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -224,7 +226,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("Finished structure analysis"); AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); @@ -379,7 +380,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { continue loop; } if (redactionLogEntry.getSectionNumber() == section.getTreeId().get(0)) { - String value = section.buildTextBlock().subSequence(new Boundary(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())).toString(); + String value = section.getTextBlock().subSequence(new Boundary(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())).toString(); if (redactionLogEntry.getValue().equalsIgnoreCase(value)) { correctFound++; } else { @@ -528,7 +529,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .stream() .filter(e -> !e.isImage()) .map(redactionLogEntry -> new Boundary(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())) - .map(boundary -> documentGraph.buildTextBlock().subSequence(boundary).toString()) + .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString()) .toList(); List valuesInRedactionLog = redactionLog.getRedactionLogEntry().stream().filter(e -> !e.isImage()).map(RedactionLogEntry::getValue).toList(); @@ -690,20 +691,27 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); + String testEntityValue1 = "Desiree"; + String testEntityValue2 = "Melanie"; RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Desiree et al"))); - assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Melanie et al."))); + assertEquals(2, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); + assertEquals(2, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count()); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."; RedactionEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get(); - String idToResize = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals("Desiree et al")).findFirst().get().getId(); + String idToResize = redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue1)) + .max(Comparator.comparingInt(RedactionLogEntry::getStartOffset)) + .get() + .getId(); List resizedPositions = expandedEntity.getRedactionPositionsPerPage() .get(0) .getRectanglePerLine() .stream() - .map(rectangle2D -> RectangleMapper.toAnnotationRectangle(rectangle2D, 3)) + .map(rectangle2D -> toAnnotationRectangle(rectangle2D, 3)) .toList(); ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() .annotationId(idToResize) @@ -724,9 +732,21 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) { fileOutputStream.write(annotateResponse.getDocument()); } - assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Desiree et al"))); - assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Melanie et al.") && !entry.lastChangeIsRemoved())); - assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals(expandedEntityKeyword))); + RedactionLogEntry resizedEntry = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get(); + assertTrue(resizedEntry.getChanges().get(resizedEntry.getChanges().size() - 1).getType().equals(ChangeType.CHANGED)); + assertEquals(idToResize, resizedEntry.getId()); + assertEquals(1, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); + assertEquals(1, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.lastChangeIsRemoved()).count()); + } + + + private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) { + + return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(), + (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(), + (float) rectangle2D.getWidth(), + -(float) rectangle2D.getHeight(), + pageNumber); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 352fc2cf..dd9ebccb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; -import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -35,7 +34,6 @@ import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class) - public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); @@ -150,7 +148,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest String entryAuthorDictionary = "Evans P.G."; dictionary.put(DICTIONARY_AUTHOR, List.of(entryAuthorDictionary)); - falsePositive.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller COMPLETION DATE:")); + falsePositive.put(DICTIONARY_PII, List.of("Dr. Alan Miller COMPLETION DATE:")); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyze(request); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java index 7fe81401..046fec99 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java @@ -37,7 +37,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -178,7 +177,16 @@ public class AnnotationService { public static List toQuadPoints(List rectangles) { - return rectangles.stream().map(RectangleMapper::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList(); + return rectangles.stream().map(AnnotationService::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList(); + } + + + private static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) { + + return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(), + redactionLogRectangle.getTopLeft().getY(), + redactionLogRectangle.getWidth(), + redactionLogRectangle.getHeight()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java index 1c3ea2a3..5599e91c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java @@ -67,7 +67,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra assertEquals("’s Donut ←", redactionEntity.getTextAfter()); assertEquals(searchTerm, redactionEntity.getValue()); assertEquals("Rule 5: Do not redact genitive CBI_authors (Entries based on Dict) ", - redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); + redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertEquals(3, redactionEntity.getIntersectingNodes().size()); assertEquals(5, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(Paragraph.class, redactionEntity.getDeepestFullyContainingNode()); @@ -86,7 +86,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra assertEquals("", redactionEntity.getTextBefore()); assertEquals(" Purity Hint", redactionEntity.getTextAfter()); assertEquals(searchTerm, redactionEntity.getValue()); - assertEquals("Rule 39: Purity Hint ", redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); + assertEquals("Rule 39: Purity Hint ", redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertEquals(3, redactionEntity.getIntersectingNodes().size()); assertEquals(6, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(Headline.class, redactionEntity.getDeepestFullyContainingNode()); @@ -105,7 +105,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra assertEquals("", redactionEntity.getTextBefore()); assertEquals("", redactionEntity.getTextAfter()); assertEquals(searchTerm, redactionEntity.getValue()); - assertEquals("Rule 6-11 (Authors Table) ", redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); + assertEquals("Rule 6-11 (Authors Table) ", redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertEquals(5, redactionEntity.getIntersectingNodes().size()); assertEquals(15, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(TableCell.class, redactionEntity.getDeepestFullyContainingNode()); @@ -131,14 +131,14 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra Document document = buildGraph("files/new/crafted document"); Table table = (Table) document.getDocumentTree()// - .streamAllEntriesInOrder()// + .allEntriesInOrder()// .filter(entry -> entry.getType().equals(NodeType.TABLE))// .map(DocumentTree.Entry::getNode)// .findFirst().orElseThrow(); assertEquals(5, table.getNumberOfCols()); assertEquals(4, table.getNumberOfRows()); assertEquals(5, table.streamHeaders().toList().size()); - CharSequence firstHeader = table.streamHeadersForCell(1, 1).map(TableCell::buildTextBlock).map(TextBlock::getSearchText).findFirst().orElseThrow(); + CharSequence firstHeader = table.streamHeadersForCell(1, 1).map(TableCell::getTextBlock).map(TextBlock::getSearchText).findFirst().orElseThrow(); assertEquals("Author(s)", firstHeader.toString().stripTrailing()); } @@ -161,7 +161,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); Table table = (Table) document.getDocumentTree() - .streamAllEntriesInOrder() + .allEntriesInOrder() .filter(entry -> entry.getNode().getPages().stream().anyMatch(page -> page.getNumber() == 22)) .filter(entry -> entry.getType().equals(NodeType.TABLE)) .map(DocumentTree.Entry::getNode) @@ -170,7 +170,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra assertEquals(5, table.getNumberOfCols()); assertEquals(14, table.getNumberOfRows()); assertEquals(10, table.streamHeaders().toList().size()); - List twoHeaders = table.streamHeadersForCell(2, 1).map(TableCell::buildTextBlock).map(TextBlock::getSearchText).toList(); + List twoHeaders = table.streamHeadersForCell(2, 1).map(TableCell::getTextBlock).map(TextBlock::getSearchText).toList(); assertEquals(2, twoHeaders.size()); assertEquals("Component of residue definition: S-Metolachlor", twoHeaders.get(0).stripTrailing()); assertEquals("Method type", twoHeaders.get(1).stripTrailing()); @@ -187,7 +187,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra assertEquals("except Cranberry; Vegetable, ", redactionEntity.getTextBefore()); assertEquals(", Group 9;", redactionEntity.getTextAfter()); assertEquals("1.1.4 Evaluations carried out under other regulatory contexts ", - redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); + redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertEquals(searchTerm, redactionEntity.getValue()); assertEquals(3, redactionEntity.getIntersectingNodes().size()); assertEquals(5, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage()); @@ -215,7 +215,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra assertEquals("2.6.1 Summary of ", redactionEntity.getTextBefore()); assertEquals(" and excretion in", redactionEntity.getTextAfter()); assertEquals("2.6.1 Summary of absorption, distribution, metabolism and excretion in mammals ", - redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); + redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertEquals(searchTerm, redactionEntity.getValue()); assertEquals(3, redactionEntity.getIntersectingNodes().size()); assertEquals(4, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage()); @@ -238,7 +238,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra assertEquals(searchTerm, redactionEntity.getValue()); assertEquals(4, redactionEntity.getIntersectingNodes().size()); assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ", - redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); + redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertTrue(redactionEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 54)); assertEquals(26, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage()); @@ -252,7 +252,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, RedactionEntity redactionEntity) { List paragraphStart = redactionEntity.getIntersectingNodes().stream()// - .map(SemanticNode::buildTextBlock)// + .map(SemanticNode::getTextBlock)// .map(textBlock -> textBlock.indexOf(searchTerm, redactionEntity.getDeepestFullyContainingNode().getBoundary().start()))// .toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java index 7d435a46..48bcc90c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java @@ -22,10 +22,10 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati @Disabled public void visualizeMetolachlor() { - String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; + String filename = "files/Trinexapac/93 Trinexapac-ethyl_RAR_03_Volume_3CA_B-1_2017-03-31"; Document document = buildGraph(filename); - TextBlock textBlock = document.buildTextBlock(); + TextBlock textBlock = document.getTextBlock(); visualizeSemanticNodes(filename, document, textBlock); } @@ -39,7 +39,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati String filename = "files/Fludioxonil/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21"; Document document = buildGraph(filename); - TextBlock textBlock = document.buildTextBlock(); + TextBlock textBlock = document.getTextBlock(); visualizeSemanticNodes(filename, document, textBlock); } @@ -53,7 +53,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati String filename = "files/new/test1S1T1"; Document document = buildGraph(filename); - TextBlock textBlock = document.buildTextBlock(); + TextBlock textBlock = document.getTextBlock(); visualizeSemanticNodes(filename, document, textBlock); } @@ -67,7 +67,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati String filename = "files/new/crafted document"; Document document = buildGraph(filename); - TextBlock textBlock = document.buildTextBlock(); + TextBlock textBlock = document.getTextBlock(); visualizeSemanticNodes(filename, document, textBlock); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java new file mode 100644 index 00000000..09b7be50 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java @@ -0,0 +1,162 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import static java.util.stream.Collectors.toMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.drools.io.ClassPathResource; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import com.google.common.base.Functions; +import com.google.common.collect.Sets; +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.RedactionLogEntryAdapter; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; +import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; + +import lombok.SneakyThrows; + +public class MigrationPocTest extends BuildDocumentIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules.drl"); + + @Autowired + private RedactionLogEntryAdapter redactionLogAdapter; + @Autowired + private RedactionLogCreatorService redactionLogCreatorService; + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + @SneakyThrows + public void testMigration() { + + AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf"); + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + AnalyzeResult result = analyzeService.analyze(request); + var newRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var originalRedactionLog = getOriginalRedactionLog(); + + // IMPORTANT: always use the graph which is mapped from the DocumentData, since rounding errors occur during storage. + Document document = DocumentGraphMapper.toDocumentGraph(DocumentData.fromDocument(buildGraph("files/new/crafted document.pdf"))); + Set migratedEntities = redactionLogAdapter.toRedactionEntity(originalRedactionLog, document).collect(Collectors.toSet()); + + var migratedRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID); + + Map migratedIds = migratedRedactionLogEntries.stream().collect(toMap(RedactionLogEntry::getId, Functions.identity())); + Map newIds = newRedactionLog.getRedactionLogEntry().stream().collect(toMap(RedactionLogEntry::getId, Functions.identity())); + + logPrecision(migratedIds, newIds); + logRecall(migratedIds, newIds); + + assertEquals(originalRedactionLog.getRedactionLogEntry().size(), migratedEntities.size()); + } + + + private static void logPrecision(Map migratedIds, Map newIds) { + + var precision = computePrecision(migratedIds, newIds); + System.out.printf("precision %.2f\n", precision); + System.out.println("New Entries"); + getAddedEntries(migratedIds, newIds).forEach(System.out::println); + assertTrue(precision > 0.9); + System.out.println(); + } + + + private static void logRecall(Map migratedIds, Map newIds) { + + var recall = computeRecall(migratedIds, newIds); + System.out.printf("recall %.2f\n", recall); + System.out.println("Missing entries"); + getMissingEntries(migratedIds, newIds).forEach(System.out::println); + assertTrue(recall > 0.9); + System.out.println(); + } + + + private static Stream getMissingEntries(Map migratedIds, Map newIds) { + + return migratedIds.entrySet().stream().filter(entry -> !newIds.containsKey(entry.getKey())).map(Map.Entry::getValue); + } + + + private static Stream getAddedEntries(Map migratedIds, Map newIds) { + + return newIds.entrySet().stream().filter(entry -> !migratedIds.containsKey(entry.getKey())).map(Map.Entry::getValue); + } + + + private static double computePrecision(Map migratedIds, Map newIds) { + + return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) newIds.size(); + } + + + private static double computeRecall(Map migratedIds, Map newIds) { + + return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) migratedIds.size(); + } + + + private static RedactionLog getOriginalRedactionLog() throws IOException { + + RedactionLog originalRedactionLog; + try (var inputStream = new ClassPathResource("files/migration/legacy_redactionlog.json").getInputStream()) { + originalRedactionLog = ObjectMapperFactory.create().readValue(inputStream, RedactionLog.class); + } + return originalRedactionLog; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java index be98b9ad..6615208c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java @@ -178,7 +178,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration Section section = document.getMainSections().get(8); start = System.currentTimeMillis(); for (int i = 0; i < n; i++) { - section.buildTextBlock(); + section.getTextBlock(); } durationMillis = ((float) (System.currentTimeMillis() - start)); System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); @@ -186,7 +186,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration SemanticNode paragraph = document.getDocumentTree().getEntryById(List.of(8, 1)).getNode(); start = System.currentTimeMillis(); for (int i = 0; i < n; i++) { - paragraph.buildTextBlock(); + paragraph.getTextBlock(); } durationMillis = ((float) (System.currentTimeMillis() - start)); System.out.printf("%d calls of buildTextBlock() on paragraph took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index 421547d5..5ce8f7e5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -130,7 +130,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { List cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList(); Stream cbiAddress = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) - .map(boundary -> new NerEntities.NerEntity(document.buildTextBlock().subSequence(boundary).toString(), boundary, "CBI_address")); + .map(boundary -> new NerEntities.NerEntity(document.getTextBlock().subSequence(boundary).toString(), boundary, "CBI_address")); return Stream.concat(cbiAuthors.stream(), cbiAddress).toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 11b0ea70..31574a5b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -3,7 +3,6 @@ package drools import static java.lang.String.format; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType; import java.util.List; import java.util.LinkedList; @@ -93,7 +92,7 @@ rule "Apply image recategorization" ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) $image: Image($id == id) then - $image.setImageType(parseImageType($imageType)); + $image.setImageType(ImageType.fromString($imageType)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl index 297cac55..a8556cd3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl @@ -3,7 +3,6 @@ package drools import static java.lang.String.format; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType; import java.util.List; import java.util.LinkedList; @@ -619,7 +618,7 @@ rule "Apply image recategorization" ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) $image: Image($id == id) then - $image.setImageType(parseImageType($imageType)); + $image.setImageType(ImageType.fromString($imageType)); end // --------------------------------------- merging rules ------------------------------------------------------------------- diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 30106e9b..79e5f495 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -3,7 +3,6 @@ package drools import static java.lang.String.format; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType; import java.util.List; import java.util.LinkedList; @@ -52,6 +51,15 @@ query "getFileAttributes" // --------------------------------------- CBI rules ------------------------------------------------------------------- +rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" + when + $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), entityType == EntityType.ENTITY) + then + RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); + falsePositive.addMatchedRule(5); + insert(falsePositive); + end + rule "0: Expand CBI_author entities with firstname initials" no-loop true when @@ -80,7 +88,7 @@ rule "0: Expand CBI_author and PII entities with salutation prefix" rule "1: Redacted because Section contains Vertebrate" when $section: Section(hasEntitiesOfType("vertebrate"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { @@ -94,7 +102,7 @@ rule "1: Redacted because Section contains Vertebrate" rule "2: Not Redacted because Section contains no Vertebrate" when $section: Section(!hasEntitiesOfType("vertebrate"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { @@ -107,8 +115,8 @@ rule "2: Not Redacted because Section contains no Vertebrate" rule "3: Do not redact Names and Addresses if no redaction Indicator is contained" when $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("no_redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + hasEntitiesOfType("no_redaction_indicator"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { @@ -121,9 +129,9 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained" when $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("no_redaction_indicator"), - hasEntitiesOfType("redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("redaction_indicator"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { @@ -137,8 +145,8 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi rule "5: Do not redact Names and Addresses if published information found" when $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("published_information"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + hasEntitiesOfType("published_information"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then List publishedInformationEntities = $section.getEntitiesOfType("published_information"); $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) @@ -193,7 +201,7 @@ rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate rule "8: Redact if must_redact entity is found" when $section: Section(hasEntitiesOfType("must_redact"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { @@ -227,7 +235,7 @@ rule "10: Redact row if row contains \"determination of residues\" and livestock "ruminants") $residueKeyword: String() from List.of("determination of residues", "determination of total residues") $table: Table(containsStringIgnoreCase($residueKeyword) - && containsStringIgnoreCase($keyword)) + && containsStringIgnoreCase($keyword)) then entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table) .forEach(keywordEntity -> insert(keywordEntity)); @@ -244,11 +252,11 @@ rule "10: Redact row if row contains \"determination of residues\" and livestock rule "11: Redact if CTL/* or BL/* was found" when - $section: Section(excludesTables, (containsString("CTL/") || containsString("BL/"))) + $section: Section(!hasTables(), (containsString("CTL/") || containsString("BL/"))) then - entityCreationService.byString("CTL/", "must_redact", EntityType.ENTITY, $section) + entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section) .forEach(mustRedactEntity -> insert(mustRedactEntity)); - entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section) + entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) .forEach(mustRedactEntity -> insert(mustRedactEntity)); $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) @@ -265,7 +273,7 @@ rule "12: Add CBI_author with \"et al.\" Regex" when $section: Section(containsString("et al.")) then - entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section) + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.setRedaction(true); entity.setRedactionReason("Author found by \"et al\" regex"); @@ -278,7 +286,7 @@ rule "12: Add CBI_author with \"et al.\" Regex" rule "13: Add recommendation for Addresses in Test Organism sections" when - $section: Section(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) + $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) then entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(redactionEntity -> { @@ -291,7 +299,7 @@ rule "13: Add recommendation for Addresses in Test Organism sections" rule "14: Add recommendation for Addresses in Test Animals sections" when - $section: Section(excludesTables, containsString("Species:"), containsString("Source:")) + $section: Section(!hasTables(), containsString("Species:"), containsString("Source:")) then entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(redactionEntity -> { @@ -385,7 +393,7 @@ rule "18: redact line between contact keywords" rule "19: Redact AUTHOR(S)" when FileAttribute(placeholder == "{fileattributes.vertebrateStudy}", value == "true") - $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:")) + $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { @@ -397,9 +405,27 @@ rule "19: Redact AUTHOR(S)" }); end -rule "20: Redact PERFORMING LABORATORY" +rule "20: Redact PERFORMING LABORATORY (Non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when - $section: Section(excludesTables, containsString("PERFORMING LABORATORY:")) + not FileAttribute(label == "Vertebrate Study", value == "Yes") + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.setRedaction(false); + laboratoryEntity.addMatchedRule(31); + laboratoryEntity.addEngine(Engine.RULE); + laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study"); + dictionary.addLocalDictionaryEntry(laboratoryEntity); + insert(laboratoryEntity); + }); + end + +rule "20: Redact PERFORMING LABORATORY" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { @@ -407,13 +433,14 @@ rule "20: Redact PERFORMING LABORATORY" authorEntity.addMatchedRule(20); authorEntity.setRedactionReason("PERFORMING LABORATORY was found"); authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + dictionary.addLocalDictionaryEntry(authorEntity); insert(authorEntity); }); end rule "21: Redact On behalf of Sequani Ltd.:" when - $section: Section(excludesTables, containsString("On behalf of Sequani Ltd.: Name Title")) + $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { @@ -427,7 +454,7 @@ rule "21: Redact On behalf of Sequani Ltd.:" rule "22: Redact On behalf of Syngenta Ltd.:" when - $section: Section(excludesTables, containsString("On behalf of Syngenta Ltd.: Name Title")) + $section: Section(!hasTables(), containsString("On behalf of Syngenta Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { @@ -439,6 +466,22 @@ rule "22: Redact On behalf of Syngenta Ltd.:" }); end + +rule "25: Redact Purity" + when + $section: Section(containsStringIgnoreCase("purity")) + then + entityCreationService.byRegex("\\bPurity:\\s*(?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section) + .forEach(entity -> { + entity.addMatchedRule(25); + entity.addEngine(Engine.RULE); + entity.setRedaction(true); + entity.setRedactionReason("Purity found"); + entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2a)"); + }); + end + + rule "26: Redact signatures" when $signature: Image(imageType == ImageType.SIGNATURE) @@ -504,11 +547,12 @@ rule "101: Redact CAS Number" end rule "102: Guidelines FileAttributes" + salience 999 when - $section: Section(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS"))) + $section: Section(!hasTables(), (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS"))) then - RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.buildTextBlock()).stream() - .map(boundary -> $section.buildTextBlock().subSequence(boundary).toString()) + RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) .forEach(fileAttribute -> insert(fileAttribute)); end @@ -589,7 +633,7 @@ rule "Apply image recategorization" ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) $image: Image($id == id) then - $image.setImageType(parseImageType($imageType)); + $image.setImageType(ImageType.fromString($imageType)); end // --------------------------------------- merging rules ------------------------------------------------------------------- @@ -653,13 +697,13 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when $entity: RedactionEntity(entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $recommendation: RedactionEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then $recommendation.removeFromGraph(); retract($recommendation); end -rule "remove Entity of lower rank, when equal boundaries and entityType" +rule "remove Entity of lower rank, when boundaries interect and entityType" salience 32 when $higherRank: RedactionEntity($type: type, $entityType: entityType) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl new file mode 100644 index 00000000..47f93f29 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl @@ -0,0 +1,697 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch; +import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.HashSet; + +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import java.util.Set +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility; + +global Document document +global EntityCreationService entityCreationService +global ManualRedactionApplicationService manualRedactionApplicationService +global NerEntitiesAdapter nerEntitiesAdapter +global Dictionary dictionary + +// --------------------------------------- queries ------------------------------------------------------------------- + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +// --------------------------------------- CBI rules ------------------------------------------------------------------- + +rule "0: Expand CBI_author entities with firstname initials" + no-loop true + when + $entityToExpand: RedactionEntity(type == "CBI_author", + value.matches("[^\\s]+"), + textAfter.startsWith(" "), + anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") + ) + then + RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)"); + expandedEntity.addMatchedRule(0); + $entityToExpand.removeFromGraph(); + retract($entityToExpand); + insert(expandedEntity); + end + +rule "0: Expand CBI_author and PII entities with salutation prefix" + when + $entityToExpand: RedactionEntity((type == "CBI_author" || type == "PII"), anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + then + RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"); + expandedEntity.addMatchedRule(0); + insert(expandedEntity); + end + +rule "1: Redacted because Section contains Vertebrate" + when + $section: Section(hasEntitiesOfType("vertebrate"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.addMatchedRule(1); + redactionEntity.setRedactionReason("Vertebrate Found in this section"); + redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + }); + end + +rule "2: Not Redacted because Section contains no Vertebrate" + when + $section: Section(!hasEntitiesOfType("vertebrate"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(false); + redactionEntity.addMatchedRule(2); + redactionEntity.setRedactionReason("No Vertebrate Found in this section"); + }); + end + +rule "3: Do not redact Names and Addresses if no redaction Indicator is contained" + when + $section: Section(hasEntitiesOfType("vertebrate"), + hasEntitiesOfType("no_redaction_indicator"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(false); + redactionEntity.addMatchedRule(3); + redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator found in this section"); + }); + end + +rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained" + when + $section: Section(hasEntitiesOfType("vertebrate"), + hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("redaction_indicator"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.addMatchedRule(4); + redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator, but also redaction-indicator, found in this section"); + redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + }); + end + +rule "5: Do not redact Names and Addresses if published information found" + + when + $section: Section(hasEntitiesOfType("vertebrate"), + hasEntitiesOfType("published_information"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + List publishedInformationEntities = $section.getEntitiesOfType("published_information"); + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(false); + redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section"); + redactionEntity.addReferences(publishedInformationEntities); + }); + end + +rule "6.0: Add all Cell's with Header Author(s) as CBI_author" + when + $table: Table(hasHeader("Author(s)")) + then + $table.streamTableCellsWithHeader("Author(s)") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .forEach(redactionEntity -> { + redactionEntity.addMatchedRule(6); + redactionEntity.setRedactionReason("Author(s) header found"); + insert(redactionEntity); + }); + end + +rule "6.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No" + when + $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No")) + then + $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No")) + .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) + .forEach(authorEntity -> { + authorEntity.setRedaction(false); + authorEntity.setRedactionReason("Not redacted because it's row does not belong to a vertebrate study"); + authorEntity.setLegalBasis(""); + authorEntity.addMatchedRule(6); + }); + end + +rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes" + when + $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes")) + then + $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) + .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.setRedactionReason("Redacted because it's row belongs to a vertebrate study"); + authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + authorEntity.addMatchedRule(7); + }); + end + +rule "8: Redact if must_redact entity is found" + when + $section: Section(hasEntitiesOfType("must_redact"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.setRedactionReason("must_redact entry was found."); + redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + redactionEntity.addMatchedRule(8); + }); + end + +rule "9: Redact CBI_sponsor entities if preceded by \" batches produced at\"" + when + $sponsorEntity: RedactionEntity(type == "CBI_sponsor", textBefore.contains("batches produced at")) + then + $sponsorEntity.setRedaction(true); + $sponsorEntity.setRedactionReason("Redacted because it represents a sponsor company"); + $sponsorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + $sponsorEntity.addMatchedRule(9); + end + +rule "10: Redact row if row contains \"determination of residues\" and livestock keyword" + when + $keyword: String() from List.of("livestock", + "live stock", + "tissue", + "tissues", + "liver", + "muscle", + "bovine", + "ruminant", + "ruminants") + $residueKeyword: String() from List.of("determination of residues", "determination of total residues") + $table: Table(containsStringIgnoreCase($residueKeyword) + && containsStringIgnoreCase($keyword)) + then + entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table) + .forEach(keywordEntity -> insert(keywordEntity)); + + $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword)) + .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.setRedactionReason("Determination of residues and keyword \"" + $keyword + "\" was found."); + redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + redactionEntity.addMatchedRule(10); + }); + end + +rule "11: Redact if CTL/* or BL/* was found" + when + $section: Section(excludesTables, (containsString("CTL/") || containsString("BL/"))) + then + entityCreationService.byString("CTL/", "must_redact", EntityType.ENTITY, $section) + .forEach(mustRedactEntity -> insert(mustRedactEntity)); + entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section) + .forEach(mustRedactEntity -> insert(mustRedactEntity)); + + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.setRedactionReason("Laboratory for vertebrate studies found"); + redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + redactionEntity.addMatchedRule(11); + }); + end + +rule "12: Add CBI_author with \"et al.\" Regex" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section) + .forEach(entity -> { + entity.setRedaction(true); + entity.setRedactionReason("Author found by \"et al\" regex"); + entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + entity.addMatchedRule(12); + insert(entity); + dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); + }); + end + +rule "13: Add recommendation for Addresses in Test Organism sections" + when + $section: Section(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) + then + entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(redactionEntity -> { + redactionEntity.setRedactionReason("Line after \"Source\" in Test Organism Section"); + redactionEntity.addMatchedRule(13); + insert(redactionEntity); + }); + end + +rule "14: Add recommendation for Addresses in Test Animals sections" + + when + $section: Section(excludesTables, containsString("Species:"), containsString("Source:")) + then + entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(redactionEntity -> { + redactionEntity.setRedactionReason("Line after \"Source:\" in Test Animals Section"); + redactionEntity.addMatchedRule(14); + insert(redactionEntity); + }); + end + +// --------------------------------------- PII rules ------------------------------------------------------------------- + +rule "15: Redact all PII" + when + $pii: RedactionEntity(type == "PII", redaction == false) + then + $pii.setRedaction(true); + $pii.setRedactionReason("PII found"); + $pii.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + $pii.addMatchedRule(15); + end + +rule "16: Redact Emails by RegEx (Non vertebrate study)" + when + $section: Section(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section) + .forEach(emailEntity -> { + emailEntity.setRedaction(true); + emailEntity.setRedactionReason("Found by Email Regex"); + emailEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + emailEntity.addMatchedRule(16); + insert(emailEntity); + }); + end + +rule "17: Redact line after contact information keywords" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No.", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> { + contactEntity.setRedaction(true); + contactEntity.addMatchedRule(17); + contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword"); + contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + insert(contactEntity); + dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false); + }); + end + + +rule "18: redact line between contact keywords" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + ) + .forEach(contactEntity -> { + contactEntity.setRedaction(true); + contactEntity.addMatchedRule(18); + contactEntity.setRedactionReason("Found between contact keywords"); + contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + insert(contactEntity); + dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false); + }); + end + +rule "19: Redact AUTHOR(S)" + when + FileAttribute(placeholder == "{fileattributes.vertebrateStudy}", value == "true") + $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.addMatchedRule(19); + authorEntity.setRedactionReason("AUTHOR(S) was found"); + authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + insert(authorEntity); + }); + end + +rule "20: Redact PERFORMING LABORATORY" + when + $section: Section(excludesTables, containsString("PERFORMING LABORATORY:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.addMatchedRule(20); + authorEntity.setRedactionReason("PERFORMING LABORATORY was found"); + authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + insert(authorEntity); + }); + end + +rule "21: Redact On behalf of Sequani Ltd.:" + when + $section: Section(excludesTables, containsString("On behalf of Sequani Ltd.: Name Title")) + then + entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.addMatchedRule(21); + authorEntity.setRedactionReason("On behalf of Sequani Ltd.: Name Title was found"); + authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + insert(authorEntity); + }); + end + +rule "22: Redact On behalf of Syngenta Ltd.:" + when + $section: Section(excludesTables, containsString("On behalf of Syngenta Ltd.: Name Title")) + then + entityCreationService.betweenStrings("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.addMatchedRule(21); + authorEntity.setRedactionReason("On behalf of Syngenta Ltd.: Name Title was found"); + authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + insert(authorEntity); + }); + end + +rule "26: Redact signatures" + when + $signature: Image(imageType == ImageType.SIGNATURE) + then + $signature.setRedaction(true); + $signature.setMatchedRule(26); + $signature.setRedactionReason("Signature Found"); + $signature.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "27: Redact formulas" + when + $formula: Image(imageType == ImageType.FORMULA) + then + $formula.setRedaction(true); + $formula.setMatchedRule(27); + $formula.setRedactionReason("Formula Found"); + $formula.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "28: Redact logos" + when + $logo: Image(imageType == ImageType.LOGO) + then + $logo.setRedaction(true); + $logo.setMatchedRule(28); + $logo.setRedactionReason("Logo Found"); + $logo.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "29: Redact Dossier Redactions" + when + $dossierRedaction: RedactionEntity(type == "dossier_redactions") + then + $dossierRedaction.setRedaction(true); + $dossierRedaction.addMatchedRule(29); + $dossierRedaction.setRedactionReason("Dossier Redaction found"); + $dossierRedaction.setLegalBasis("Article 39(1)(2) of Regulation (EC) No 178/2002"); + end + +rule "30: Remove Dossier redactions if file is confidential" + when + FileAttribute(label == "Confidentiality", value == "confidential") + $dossierRedaction: RedactionEntity(type == "dossier_redactions") + then + $dossierRedaction.removeFromGraph(); + retract($dossierRedaction) + end + +rule "101: Redact CAS Number" + when + $table: Table(hasHeader("Sample #")) + then + $table.streamTableCellsWithHeader("Sample #") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY)) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.addMatchedRule(101); + redactionEntity.setRedactionReason("Sample # found in Header"); + redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + insert(redactionEntity); + }); + end + +rule "102: Guidelines FileAttributes" + when + $section: Section(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS"))) + then + RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +// --------------------------------------- NER Entities rules ------------------------------------------------------------------- + +rule "add NER Entities of type CBI_author" + salience 999 + when + nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) + then + nerEntities.streamEntitiesOfType("CBI_author") + .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) + .forEach(entity -> insert(entity)); + end + +rule "combine and add NER Entities as CBI_address" + salience 999 + when + nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY")) + then + nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) + .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) + .forEach(entity -> { + entity.addEngine(Engine.NER); + insert(entity); + }); + end + +// --------------------------------------- manual redaction rules ------------------------------------------------------------------- + +rule "Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId) + $entityToBeResized: RedactionEntity(matchesAnnotationId($id)) + then + manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($entityToBeResized); + end + +rule "Apply id removals that are valid and not in forced redactions to Entity" + salience 128 + when + IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) + then + $entityToBeRemoved.removeFromGraph(); + retract($entityToBeRemoved); + end + +rule "Apply id removals that are valid and not in forced redactions to Image" + salience 128 + when + IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $entityToBeRemoved: Image($id == id) + then + $entityToBeRemoved.setIgnored(true); + end + +rule "Apply force redaction" + salience 128 + when + ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $entityToForce: RedactionEntity(matchesAnnotationId($id)) + then + $entityToForce.setLegalBasis($legalBasis); + $entityToForce.setRedaction(true); + $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + end + +rule "Apply image recategorization" + salience 128 + when + ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) + $image: Image($id == id) + then + $image.setImageType(ImageType.fromString($imageType)); + end + +// --------------------------------------- merging rules ------------------------------------------------------------------- + +rule "remove Entity contained by Entity of same type" + salience 65 + when + $larger: RedactionEntity($type: type, $entityType: entityType) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + then + $contained.removeFromGraph(); + retract($contained); + end + +rule "merge intersecting Entities of same type" + salience 64 + when + $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) + $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) + then + $first.removeFromGraph(); + $second.removeFromGraph(); + RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + retract($first); + retract($second); + insert(mergedEntity); + end + +rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 64 + when + $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) + $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + then + $entity.removeFromGraph(); + retract($entity) + end + +rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 64 + when + $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) + $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" + salience 256 + when + $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $entity.addEngines($recommendation.getEngines()); + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 256 + when + $entity: RedactionEntity(entityType == EntityType.ENTITY) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of lower rank, when equal boundaries and entityType" + salience 32 + when + $higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary) + $lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction) + then + $lowerRank.removeFromGraph(); + retract($lowerRank); + end + +// --------------------------------------- FileAttribute Rules ------------------------------------------------------------------- + +rule "remove duplicate FileAttributes" + salience 64 + when + $fileAttribute: FileAttribute($label: label, $value: value) + $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value) + then + retract($duplicate); + end + +// --------------------------------------- local dictionary search ------------------------------------------------------------------- + +rule "run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) + .forEach(entity -> { + entity.addEngine(Engine.RULE); + insert(entity); + }); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 7db4d5b8..1a43d6cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -3,7 +3,6 @@ package drools import static java.lang.String.format; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType; import java.util.List; import java.util.LinkedList; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/legacy_redactionlog.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/legacy_redactionlog.json new file mode 100644 index 00000000..46d9a1ae --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/legacy_redactionlog.json @@ -0,0 +1 @@ +{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"2d9e1b1baaca3fac422c05c0d9721937","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":1}],"sectionNumber":45,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243351729+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"188e6ba388f6bef50e69a7fa3672768e","type":"CBI_author","value":"Mustermann","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 1/2: Redact CBI Authors based on Dict","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":81.796005,"y":294.04202},"width":60.576004,"height":-12.641998,"page":1}],"sectionNumber":2,"textBefore":"David Ksenia Max ","textAfter":" Ranya Eikenboom","comments":[],"startOffset":131,"endOffset":141,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243369301+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"8db3b591647b0106ef78667468be105b","type":"CBI_author","value":"Doe","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":131.776,"y":429.14203},"width":20.003998,"height":-12.641998,"page":1}],"sectionNumber":1,"textBefore":"F. Lastname, J. ","textAfter":", M. Mustermann","comments":[],"startOffset":67,"endOffset":70,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243371886+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"00ac3ed9b409f2ed8e9c735d98735011","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":1}],"sectionNumber":36,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243373132+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"28d8a6badb6870096775ccf4b73b0401","type":"PII","value":"David","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 1/2: Redact CBI Authors based on Dict","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":56.8,"y":307.84204},"width":29.291996,"height":-12.641998,"page":1}],"sectionNumber":2,"textBefore":"Study is No ","textAfter":" Ksenia Max","comments":[],"startOffset":114,"endOffset":119,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243374232+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"4838873220a40a2182ae7d8c22132571","type":"CBI_author","value":"Lastname M.","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":56.8,"y":415.34204},"width":63.275997,"height":-12.641998,"page":1}],"sectionNumber":1,"textBefore":"Doe, M. Mustermann ","textAfter":", Doe J.","comments":[],"startOffset":86,"endOffset":97,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243375288+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"040b78c7ac09c65797d5d8ae3df90452","type":"CBI_author","value":"Mustermann M.","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":159.66399,"y":415.34204},"width":77.36401,"height":-12.641998,"page":1}],"sectionNumber":1,"textBefore":"M., Doe J. ","textAfter":null,"comments":[],"startOffset":106,"endOffset":119,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243376289+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"354a1995d18c26c1071d7b15d77f6502","type":"CBI_author","value":"F. Lastname","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":56.8,"y":429.14203},"width":58.295994,"height":-12.641998,"page":1}],"sectionNumber":1,"textBefore":"with firstname initials ","textAfter":", J. Doe,","comments":[],"startOffset":51,"endOffset":62,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243377325+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"b9f9423e69e7d50367b18b9c01d0a133","type":"CBI_author","value":"Doe J.","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":126.076,"y":415.34204},"width":30.58799,"height":-12.641998,"page":1}],"sectionNumber":1,"textBefore":"Mustermann Lastname M., ","textAfter":" Mustermann M.","comments":[],"startOffset":99,"endOffset":105,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243378585+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"9a8f34beeeb8c30c54c40515a9d8b4a4","type":"CBI_author","value":"M. Mustermann","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":157.768,"y":429.14203},"width":77.268005,"height":-12.641998,"page":1}],"sectionNumber":1,"textBefore":"Lastname, J. Doe, ","textAfter":" Lastname M.,","comments":[],"startOffset":72,"endOffset":85,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243379537+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"9020b603ab5f447ab58f2708a03c683f","type":"CBI_address","value":"Netherlands","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 3/4: Redact (not) CBI Add/ress based on Dict","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":217.252,"y":647.642},"width":57.86403,"height":-12.641998,"page":2}],"sectionNumber":3,"textBefore":"7232 CX Warnsveld, ","textAfter":", NL Institut","comments":[],"startOffset":174,"endOffset":185,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243380532+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"d8b821220916bf878bf818e445f9978a","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":2}],"sectionNumber":46,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243381509+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"4d7e41d8f0903ccf13b56b7ddf92447f","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":2}],"sectionNumber":37,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243382661+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"af12c1348dfbf842c3f70740da4f8f1c","type":"CBI_author","value":"Melanie","reason":"Author found","matchedRule":12,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2g)","imported":false,"redacted":true,"section":"Rule 14/15: Redact and add recommendation for et al.","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":214.66,"y":455.14203},"width":39.300003,"height":-12.641998,"page":3}],"sectionNumber":12,"textBefore":"dolore magna aliqua ","textAfter":" et al.","comments":[],"startOffset":292,"endOffset":299,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243383648+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"d012b10977bd4d3cc1199323a51ba8b9","type":"CBI_author","value":"Melanie","reason":"Author found","matchedRule":12,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2g)","imported":false,"redacted":true,"section":"Rule 14/15: Redact and add recommendation for et al.","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":162.12,"y":481.235},"width":32.710007,"height":-11.535004,"page":3}],"sectionNumber":12,"textBefore":"Term “Desiree”, “","textAfter":"” and add","comments":[],"startOffset":77,"endOffset":84,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24338468+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"e56839ce73666cf9f00723b97cb59360","type":"must_redact","value":"bl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":56.8,"y":633.84204},"width":15.312,"height":-12.641998,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":163,"endOffset":165,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24338574+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"b0ee886cd35ead9073665e931fb19073","type":"CBI_author","value":"Desiree","reason":"Author found","matchedRule":12,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2g)","imported":false,"redacted":true,"section":"Rule 14/15: Redact and add recommendation for et al.","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":115.27,"y":481.235},"width":30.700005,"height":-11.535004,"page":3}],"sectionNumber":12,"textBefore":"Redact Term “","textAfter":"”, “Melanie” and","comments":[],"startOffset":66,"endOffset":73,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243386688+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"844c845dd528a3306ddaa68bfca78686","type":"must_redact","value":"ctl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":229.79291,"y":705.9678},"width":27.382202,"height":-13.867798,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":27,"endOffset":30,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243387658+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"4e8ed7c89ef19c9fd0d068ced5169639","type":"CBI_author","value":"Desiree","reason":"Author found","matchedRule":12,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2g)","imported":false,"redacted":true,"section":"Rule 14/15: Redact and add recommendation for et al.","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":324.28015,"y":468.94202},"width":36.57602,"height":-12.641998,"page":3}],"sectionNumber":12,"textBefore":"consectetur adipiscing elit ","textAfter":" et al","comments":[],"startOffset":212,"endOffset":219,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243388695+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"e8b8848ca52246fc70d30a8dcd9dbec1","type":"vertebrate","value":"mouse","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 16/17: Add recommendation for Addresses in Test Organism/Animals\nsections","color":[1.0,0.52156866,0.96862745],"positions":[{"topLeft":{"x":209.716,"y":356.74203},"width":32.60399,"height":-12.641998,"page":3}],"sectionNumber":13,"textBefore":null,"textAfter":null,"comments":[],"startOffset":169,"endOffset":174,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243389668+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"6297da8bcb4ad9fda3729af2bfd0aa61","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":3}],"sectionNumber":38,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243390614+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"45486705028b69c37a26119aa2a97ad0","type":"must_redact","value":"ctl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":56.8,"y":592.442},"width":22.607998,"height":-12.641998,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":208,"endOffset":211,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243391561+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"6a22b564909681c1ef1715699b814b64","type":"must_redact","value":"bl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":278.87503,"y":705.9678},"width":18.78122,"height":-13.867798,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":34,"endOffset":36,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243392552+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"f334f16549061c2be9f8c872deb762fa","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":3}],"sectionNumber":47,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24339345+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"22e5997a9d2dd5b90c394b41df46bdb2","type":"vertebrate","value":"mouse","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 16/17 (additional) negative Test; missing second Key","color":[1.0,0.52156866,0.96862745],"positions":[{"topLeft":{"x":348.59204,"y":148.84204},"width":32.604004,"height":-12.642029,"page":3}],"sectionNumber":15,"textBefore":null,"textAfter":null,"comments":[],"startOffset":203,"endOffset":208,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243394515+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"e9092f4ef1d0630c46a1344bff41ee56","type":"must_redact","value":"bl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":56.8,"y":620.04205},"width":15.312,"height":-12.641998,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":191,"endOffset":193,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243395506+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"c4c93e8f52891ea116181b369eee1c0f","type":"must_redact","value":"bl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":56.8,"y":551.04205},"width":15.312,"height":-12.641998,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":336,"endOffset":338,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243396483+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"0fb57c171645ae8960c90ea37597c470","type":"vertebrate","value":"mouse","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 16/17 (additional) negative Test; missing second Key","color":[1.0,0.52156866,0.96862745],"positions":[{"topLeft":{"x":336.9161,"y":162.64203},"width":32.603973,"height":-12.642029,"page":3}],"sectionNumber":15,"textBefore":null,"textAfter":null,"comments":[],"startOffset":115,"endOffset":120,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243397421+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"4b2419a4ed2bd46a1bcd78ea96809746","type":"must_redact","value":"bl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":56.8,"y":564.84204},"width":15.312,"height":-12.641998,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":295,"endOffset":297,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243398402+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"397ebcd721fb1ce243eaa6f08f3794ec","type":"must_redact","value":"ctl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":56.8,"y":578.642},"width":22.607998,"height":-12.641998,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":250,"endOffset":253,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24339936+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"80f105bfe099323268836c343e139c78","type":"must_redact","value":"ctl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":56.8,"y":647.642},"width":22.607998,"height":-12.641998,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":145,"endOffset":148,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243400289+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"15255a149dec8caf8594a428ad07ea76","type":"must_redact","value":"ctl","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 12/13: Redact/Hint if CTL or BL was found","color":[0.98039216,0.7058824,0.7529412],"positions":[{"topLeft":{"x":56.8,"y":661.442},"width":22.607998,"height":-12.641998,"page":3}],"sectionNumber":11,"textBefore":null,"textAfter":null,"comments":[],"startOffset":116,"endOffset":119,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243401307+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"7b2c1111306baf6a105aee47f7538f1d","type":"PII","value":"Naka-27 Aomachi, Nomi, Ishikawa 923-1101, Japan, JP","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 19/20: Redacted PII Personal Identification Information based on Dict","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":56.8,"y":494.14203},"width":269.37607,"height":-12.641998,"page":4}],"sectionNumber":17,"textBefore":"Study is No ","textAfter":" Sude Halide","comments":[],"startOffset":146,"endOffset":197,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243402236+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"9242beedf953556f50cd48c6ebec87d8","type":"CBI_author","value":"Liu","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 18: Do not redact Names and Addresses if Published Information\nfound","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":449.4521,"y":601.642},"width":16.596039,"height":-12.641998,"page":4}],"sectionNumber":16,"textBefore":"Zhou Mah, Ning ","textAfter":", Lei W.","comments":[],"startOffset":494,"endOffset":497,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243403228+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"58d1b1a3cec2f846fbf47828b1975b3f","type":"PII","value":"dinther@comcast.net","reason":"PII (Personal Identification Information) found","matchedRule":15,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 21/22: Redact Emails by RegEx","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":333.89212,"y":303.84204},"width":101.98816,"height":-12.641998,"page":4}],"sectionNumber":18,"textBefore":"pariatur. Excepteur sint ","textAfter":" occaecat cupidatat","comments":[],"startOffset":262,"endOffset":281,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243404231+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"5dcb245404303d9160bc2b310d81847f","type":"PII","value":"gordonjcp@msn.com","reason":"PII (Personal Identification Information) found","matchedRule":15,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 21/22: Redact Emails by RegEx","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":404.96823,"y":317.64203},"width":103.308075,"height":-12.641998,"page":4}],"sectionNumber":18,"textBefore":"reprehenderit in voluptate ","textAfter":" velit esse","comments":[],"startOffset":178,"endOffset":195,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243405288+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"e723fdeac531c28ea2ab172b7e537b3f","type":"PII","value":"Sude Halide Nurullah","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 19/20: Redacted PII Personal Identification Information based on Dict","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":56.8,"y":480.34204},"width":104.556015,"height":-12.641998,"page":4}],"sectionNumber":17,"textBefore":"923-1101, Japan, JP ","textAfter":" Özgür U.","comments":[],"startOffset":198,"endOffset":218,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243406335+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"206e7a9a7eb04b39b3fa093e57e51aab","type":"PII","value":"Xinyi Y. Tao","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 19/20: Redacted PII Personal Identification Information based on Dict","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":56.8,"y":425.14203},"width":60.599987,"height":-12.641998,"page":4}],"sectionNumber":17,"textBefore":"C. J. Alfred ","textAfter":" Clara Siegfried","comments":[],"startOffset":257,"endOffset":269,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243407352+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"19cbb3c4ef9f60dde6a10bfd21519822","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":4}],"sectionNumber":39,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243408254+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"3f5859286e2801eb163ed26b874ea072","type":"CBI_author","value":"Wong","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 18: Do not redact Names and Addresses if Published Information\nfound","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":269.51196,"y":601.642},"width":28.29602,"height":-12.641998,"page":4}],"sectionNumber":16,"textBefore":"Jun K., Tu ","textAfter":", Qiang Suen,","comments":[],"startOffset":461,"endOffset":465,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243409223+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"00642e9a0d683818870ab4abf3d99132","type":"PII","value":"B. Rahim","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 19/20: Redacted PII Personal Identification Information based on Dict","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":56.8,"y":452.74203},"width":45.91199,"height":-12.641998,"page":4}],"sectionNumber":17,"textBefore":"Özgür U. Reyhan ","textAfter":" C. J.","comments":[],"startOffset":235,"endOffset":243,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24341029+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"ddbe416a72b9d5a0ea006581c643cee1","type":"PII","value":"kawasaki@me.com","reason":"PII (Personal Identification Information) found","matchedRule":15,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 21/22: Redact Emails by RegEx","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":138.06398,"y":290.04205},"width":93.99602,"height":-12.642029,"page":4}],"sectionNumber":18,"textBefore":"proident, sunt in ","textAfter":" culpa qui","comments":[],"startOffset":323,"endOffset":338,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243411232+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"12c20c1c1f6fd8daa5b12e503b0e4940","type":"PII","value":"library@outlook.com","reason":"PII (Personal Identification Information) found","matchedRule":15,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 21/22: Redact Emails by RegEx","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":171.748,"y":317.64203},"width":103.296036,"height":-12.641998,"page":4}],"sectionNumber":18,"textBefore":"irure dolor in ","textAfter":" reprehenderit in","comments":[],"startOffset":131,"endOffset":150,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243412189+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"15670cc8b24fa89e657ecc52988e21ba","type":"CBI_author","value":"Huang","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 18: Do not redact Names and Addresses if Published Information\nfound","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":507.04016,"y":601.642},"width":31.991943,"height":-12.641998,"page":4}],"sectionNumber":16,"textBefore":"Liu, Lei W. ","textAfter":", Ru X.","comments":[],"startOffset":506,"endOffset":511,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243413173+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"b8f28659e26466db1d6341d637ff688e","type":"CBI_author","value":"Wu","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 18: Do not redact Names and Addresses if Published Information\nfound","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":88.192,"y":587.84204},"width":16.800003,"height":-12.641998,"page":4}],"sectionNumber":16,"textBefore":"Huang, Ru X. ","textAfter":null,"comments":[],"startOffset":519,"endOffset":521,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.2434141+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"0e72674bb524dff6c1d9e63f164a070b","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":4}],"sectionNumber":48,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243415043+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"1fdfac58c3648655cc8b1ed89ffe8370","type":"PII","value":"Özgür U. Reyhan","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 19/20: Redacted PII Personal Identification Information based on Dict","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":56.8,"y":466.54202},"width":84.192,"height":-12.641998,"page":4}],"sectionNumber":17,"textBefore":"Sude Halide Nurullah ","textAfter":" B. Rahim","comments":[],"startOffset":219,"endOffset":234,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243416046+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"d380934c73ae7308cc323de5047f4290","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":5}],"sectionNumber":40,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243417163+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"2f429d40393e13a0a09e239d0edeb077","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":5}],"sectionNumber":49,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243429972+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"db41635251ff55a9b7b30f5f97035ddd","type":"PII","value":"493 1223 4592","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":341.44302,"y":473.68152},"width":53.991028,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"1432 8933 Contact: ","textAfter":" European contact:","comments":[],"startOffset":664,"endOffset":677,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24347322+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"ebd435e3aa7d2cfa8a80dbaed47a0423","type":"PII","value":"Central Research Industry","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":362.22406,"y":628.88153},"width":93.77109,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"Contact point: ","textAfter":" Phone: +49","comments":[],"startOffset":243,"endOffset":268,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243474339+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"815ad6c1369cbc76c97f4dd571db1aaa","type":"PII","value":"\")","reason":"Contact information was found","matchedRule":16,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":490.14966,"y":705.9678},"width":11.378723,"height":-13.867798,"page":5}],"sectionNumber":22,"textBefore":"(contains \"Contact point:","textAfter":" Redact when","comments":[],"startOffset":64,"endOffset":66,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243475348+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"5b1a9187d0846eaf96eb7995e9979148","type":"PII","value":"+274 34223331","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":350.83902,"y":494.3815},"width":56.79901,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"1432 8990 Telephone: ","textAfter":" Phone No.","comments":[],"startOffset":616,"endOffset":629,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243476405+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"7bda87d0a1c95e317d5a60842990be8c","type":"PII","value":"maximiliamschmitt@t-online.de","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":336.43906,"y":556.4815},"width":116.82919,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"Email: maximiliamschmitt@arcor.de e-mail: ","textAfter":" E-mail address:","comments":[],"startOffset":427,"endOffset":456,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.2434774+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"7e3055bdc31d168bc8945823884a9839","type":"PII","value":"+81 764770164","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":327.06104,"y":597.88153},"width":56.889008,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"2311 560 Tel.: ","textAfter":" Tel: +81","comments":[],"startOffset":323,"endOffset":336,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243478491+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"bfbb43e30ff21aff5c54fb5c81c14275","type":"PII","value":"+49 2113 2311 563","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":335.98,"y":618.5815},"width":69.68704,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"Research Industry Phone: ","textAfter":" Fax: +49","comments":[],"startOffset":276,"endOffset":293,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24347946+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"34e18828e7b568e40fcee9526abdc9c4","type":"PII","value":"+81 6653 44563","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":324.86502,"y":587.4815},"width":59.085022,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"+81 764770164 Tel: ","textAfter":" E-mail: Seriknowmobil@co.uk","comments":[],"startOffset":342,"endOffset":356,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243480383+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"d25c1b20aa17503f8c0d9fd008ae2738","type":"PII","value":"Emilia Lockhart","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":382.0601,"y":452.9815},"width":58.69806,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"Institute Alternative contact: ","textAfter":" Alternative contact:","comments":[],"startOffset":744,"endOffset":759,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243481349+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"45322c38892a816d3a37e29352b47f1a","type":"PII","value":"Maximiliam Schmitt","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":341.44302,"y":535.7815},"width":74.59213,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"address: example@mail.com Contact: ","textAfter":" Telephone number:","comments":[],"startOffset":499,"endOffset":517,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243482307+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"b594d911b3482334dcb695b1340acd78","type":"PII","value":"example@mail.com","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":367.21008,"y":546.0815},"width":72.387085,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"maximiliamschmitt@t-online.de E-mail address: ","textAfter":" Contact: Maximiliam","comments":[],"startOffset":473,"endOffset":489,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24348326+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"98c722ff843e5251195055e66a031093","type":"PII","value":"European Central Institute","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":376.09305,"y":463.2815},"width":94.86011,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"4592 European contact: ","textAfter":" Alternative contact:","comments":[],"startOffset":696,"endOffset":722,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243484217+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"212051e38cb5f80526f144684fc80fa4","type":"PII","value":"+49 2113 2311 560","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":326.98,"y":608.1815},"width":69.68704,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"2311 563 Fax: ","textAfter":" Tel.: +81","comments":[],"startOffset":299,"endOffset":316,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243485212+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"044298f0db79bbc360bdaad3d1411251","type":"PII","value":"+274 1432 8991","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":364.03302,"y":515.08154},"width":59.085022,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"+27414328992 Telephone No: ","textAfter":" Fax number:","comments":[],"startOffset":563,"endOffset":577,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243486157+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"9a0114d10f74939b10002106806d7422","type":"PII","value":"Seriknowmobil@co.uk","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":337.94205,"y":577.1815},"width":83.52905,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"6653 44563 E-mail: ","textAfter":" Email: maximiliamschmitt@arcor.de","comments":[],"startOffset":365,"endOffset":384,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243542304+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"138b4df6e48ed3fe06837499bf57d1b7","type":"PII","value":"+274 1432 8990","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":356.75204,"y":504.68152},"width":58.995026,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"8991 Fax number: ","textAfter":" Telephone: +274","comments":[],"startOffset":590,"endOffset":604,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243544144+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"b4ce598ebbd19248881e990b0be749c6","type":"PII","value":"maximiliamschmitt@arcor.de","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":334.94504,"y":566.7815},"width":106.83017,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"E-mail: Seriknowmobil@co.uk Email: ","textAfter":" e-mail: maximiliamschmitt@t-online.de","comments":[],"startOffset":392,"endOffset":418,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243545644+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"d6dd1bb84c826b4f4e89f93432866e03","type":"PII","value":"+27414328992","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 23/24: Redact contact information (contains \"Contact point:\")","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":380.51205,"y":525.38153},"width":54.593994,"height":-10.981506,"page":5}],"sectionNumber":20,"textBefore":"Schmitt Telephone number: ","textAfter":" Telephone No:","comments":[],"startOffset":536,"endOffset":548,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243547115+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"88c02bfa6be47a6aad387f91ea909d50","type":"PII","value":"+274 1432 8990","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":356.75204,"y":309.0815},"width":58.995026,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"8991 Fax number: ","textAfter":" Telephone: +274","comments":[],"startOffset":590,"endOffset":604,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24354862+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"6016a24511775bacabecb82fbec9d9b8","type":"PII","value":"Central Research Industry","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":362.22406,"y":433.2815},"width":93.77109,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"Contact point: ","textAfter":" Phone: +49","comments":[],"startOffset":243,"endOffset":268,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24355005+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"32e8c40a25a6cdcf9ddd9fb41c3dc8c8","type":"PII","value":"+82 122 34188","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":164.8,"y":537.142},"width":72.696,"height":-12.641998,"page":6}],"sectionNumber":26,"textBefore":"South Korea Phone: ","textAfter":" Fax: +82","comments":[],"startOffset":333,"endOffset":346,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243551366+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"07f0148ffec4d215e6daf04fb11d7a97","type":"PII","value":"493 1223 4592","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":341.44302,"y":278.08148},"width":53.991028,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"1432 8933 Contact: ","textAfter":" European contact:","comments":[],"startOffset":664,"endOffset":677,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243552681+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"2206b5d18ef3649095d6a8285e8ae510","type":"PII","value":"Riddley Scott","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":164.8,"y":592.34204},"width":65.59201,"height":-12.641998,"page":6}],"sectionNumber":26,"textBefore":"Corporation Contact point: ","textAfter":" Address: 359-21","comments":[],"startOffset":256,"endOffset":269,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243553946+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"174b5962e6b66906ed96341bbf8da898","type":"PII","value":"+275 5678 1234 132","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":164.8,"y":468.14203},"width":99.696,"height":-12.641998,"page":6}],"sectionNumber":26,"textBefore":"be redacted Tel.: ","textAfter":null,"comments":[],"startOffset":501,"endOffset":519,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243555257+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"8d127621a973c216cc2f682ced59348c","type":"PII","value":"This is a special case, everything between this and the next keyword should be redacted","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":164.8,"y":495.74203},"width":282.74426,"height":-12.641998,"page":6},{"topLeft":{"x":164.8,"y":481.94202},"width":134.86809,"height":-12.641998,"page":6}],"sectionNumber":26,"textBefore":"E-mail: food-industry@korea.com Contact: ","textAfter":" Tel.: +275","comments":[],"startOffset":407,"endOffset":494,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243556691+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"baac21659a54cf4b88fecfe9488afe23","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":6}],"sectionNumber":50,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24355812+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"a1c4bee8f2c7ddc33d1269c31f199f34","type":"PII","value":"food-industry@korea.com","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":164.8,"y":509.54202},"width":126.60005,"height":-12.641998,"page":6}],"sectionNumber":26,"textBefore":"122 34180 E-mail: ","textAfter":" Contact: This","comments":[],"startOffset":374,"endOffset":397,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243559424+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"f33c9953e285b806379220eefb2bf8f9","type":"PII","value":"+274 34223331","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":350.83902,"y":298.7815},"width":56.79901,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"1432 8990 Telephone: ","textAfter":" Phone No.","comments":[],"startOffset":616,"endOffset":629,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243560877+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"33262ab2cf966c69daea340311ea2c93","type":"PII","value":"Maximiliam Schmitt","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":341.44302,"y":340.18152},"width":74.59213,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"address: example@mail.com Contact: ","textAfter":" Telephone number:","comments":[],"startOffset":499,"endOffset":517,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243562313+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"11a1cc600741966ba67ebc7d194a9092","type":"PII","value":"example@mail.com","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":367.21008,"y":350.4815},"width":72.387085,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"maximiliamschmitt@t-online.de E-mail address: ","textAfter":" Contact: Maximiliam","comments":[],"startOffset":473,"endOffset":489,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243563676+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"3dd0fddae07b2ec44ef5877242b774e7","type":"PII","value":"+274 1432 8991","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":364.03302,"y":319.4815},"width":59.085022,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"+27414328992 Telephone No: ","textAfter":" Fax number:","comments":[],"startOffset":563,"endOffset":577,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243565102+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"d99abab08d4e42148f37ab781d0a9536","type":"PII","value":"maximiliamschmitt@t-online.de","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":336.43906,"y":360.8815},"width":116.82919,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"Email: maximiliamschmitt@arcor.de e-mail: ","textAfter":" E-mail address:","comments":[],"startOffset":427,"endOffset":456,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243566611+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"7363de5ea8ddc4aa939332b5af4a22a2","type":"PII","value":"+81 764770164","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":327.06104,"y":402.2815},"width":56.889008,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"2311 560 Tel.: ","textAfter":" Tel: +81","comments":[],"startOffset":323,"endOffset":336,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243567811+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"727c450c0a0944c60a55625a4fafcff1","type":"PII","value":"+49 2113 2311 563","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":335.98,"y":422.9815},"width":69.68704,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"Research Industry Phone: ","textAfter":" Fax: +49","comments":[],"startOffset":276,"endOffset":293,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243568991+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"ffad12ecac583473bddf7ff3057f2123","type":"PII","value":"+81 6653 44563","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":324.86502,"y":391.8815},"width":59.085022,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"+81 764770164 Tel: ","textAfter":" E-mail: Seriknowmobil@co.uk","comments":[],"startOffset":342,"endOffset":356,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243570197+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"74be768aad40501c90e30ad02d2f9177","type":"PII","value":"European Central Institute","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":376.09305,"y":267.68152},"width":94.86011,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"4592 European contact: ","textAfter":" Alternative contact:","comments":[],"startOffset":696,"endOffset":722,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243571395+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"65e0a06c7164a9c631865c0e56b29518","type":"PII","value":"+27414328992","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":380.51205,"y":329.7815},"width":54.593994,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"Schmitt Telephone number: ","textAfter":" Telephone No:","comments":[],"startOffset":536,"endOffset":548,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24357247+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"c658bcdcbcd8cf678b79cc715c024c70","type":"PII","value":"+82 122 34180","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":164.8,"y":523.34204},"width":72.696,"height":-12.641998,"page":6}],"sectionNumber":26,"textBefore":"122 34188 Fax: ","textAfter":" E-mail: food-industry@korea.com","comments":[],"startOffset":352,"endOffset":365,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243573586+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"46e0173e2da55fc771e151b6f90d08f3","type":"PII","value":"maximiliamschmitt@arcor.de","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":334.94504,"y":371.18152},"width":106.83017,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"E-mail: Seriknowmobil@co.uk Email: ","textAfter":" e-mail: maximiliamschmitt@t-online.de","comments":[],"startOffset":392,"endOffset":418,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243574996+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"8332ab33f0b7ba918b086a7c5f1f25e6","type":"PII","value":"+49 2113 2311 560","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":326.98,"y":412.5815},"width":69.68704,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"2311 563 Fax: ","textAfter":" Tel.: +81","comments":[],"startOffset":299,"endOffset":316,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243576611+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"2d8809c60064fc8e389a92409380159d","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":6}],"sectionNumber":41,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.2435779+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"84311f71dd0c3babb2b34ae074a00794","type":"PII","value":"Seriknowmobil@co.uk","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":337.94205,"y":381.5815},"width":83.52905,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"6653 44563 E-mail: ","textAfter":" Email: maximiliamschmitt@arcor.de","comments":[],"startOffset":365,"endOffset":384,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243579207+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"1e61596a94430ba7e602abaa1d226783","type":"PII","value":"Emilia Lockhart","reason":"Applicant information was found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Table in: Rule 25/26: Redact contact information (contains \"Applicant\" as Headline\nor Text)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":382.0601,"y":257.38153},"width":58.69806,"height":-10.981506,"page":6}],"sectionNumber":24,"textBefore":"Institute Alternative contact: ","textAfter":" Alternative contact:","comments":[],"startOffset":744,"endOffset":759,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243580378+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"09197adc6d8b3e60f87f67f51310c99c","type":"PII","value":"+82 122 34180","reason":"Contact information was found","matchedRule":16,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 27/28: Redact contact Information (contains Producer)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":128.8,"y":527.942},"width":72.696,"height":-12.641998,"page":7}],"sectionNumber":27,"textBefore":"122 34188 Fax: ","textAfter":" E-mail: pharma-industry@korea.com","comments":[],"startOffset":335,"endOffset":348,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243581545+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"fb1bb307f2cd487256b65b767cefa6e1","type":"PII","value":"pharma-industry@korea.com","reason":"Contact information was found","matchedRule":16,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 27/28: Redact contact Information (contains Producer)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":128.8,"y":514.142},"width":140.59203,"height":-12.641998,"page":7}],"sectionNumber":27,"textBefore":"122 34180 E-mail: ","textAfter":null,"comments":[],"startOffset":357,"endOffset":382,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243582928+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"b626ba58f50e7a3523c58d77c2d5843f","type":"CBI_author","value":"Grant","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 29/30/31/32: If Text contains \"AUTHORS:\" and \"COMPLETION\nDATES\" but not \"STUDY COMPLETION DATES\", then Redact between\nboth","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":244.38402,"y":346.64203},"width":27.312027,"height":-12.641998,"page":7}],"sectionNumber":28,"textBefore":"AUTHOR(S): Dr. Alan ","textAfter":" COMPLETION DATE:","comments":[],"startOffset":235,"endOffset":240,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243584355+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"1389caf8701b9944fa842c4772a84a53","type":"PII","value":"Jill Valentine","reason":"Contact information was found","matchedRule":16,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 27/28: Redact contact Information (contains Producer)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":128.8,"y":596.942},"width":62.676025,"height":-12.641998,"page":7}],"sectionNumber":27,"textBefore":"Umbrella Corporation Contact: ","textAfter":" Address: 359-21","comments":[],"startOffset":238,"endOffset":252,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243587301+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"f163afe507f0eca1cde5fa049a6c870e","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":7}],"sectionNumber":42,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243596895+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"2c4dd52bc76fd344ace84ae6847b3a2e","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":7}],"sectionNumber":51,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243598499+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"5adc8b81ad610d4739cdf69b0773d8c6","type":"PII","value":"+82 122 34188","reason":"Contact information was found","matchedRule":16,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 27/28: Redact contact Information (contains Producer)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":128.8,"y":541.742},"width":72.696,"height":-12.641998,"page":7}],"sectionNumber":27,"textBefore":"South Korea Phone: ","textAfter":" Fax: +82","comments":[],"startOffset":316,"endOffset":329,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243599853+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"83e61843f54422d17470e00e3366988f","type":"CBI_author","value":"Grant","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 29/30/31/32: (additional) negative Test for Study completion dates","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":280.38403,"y":195.54205},"width":27.312012,"height":-12.642029,"page":7}],"sectionNumber":29,"textBefore":"AUTHOR(S): Dr. Alan ","textAfter":" STUDY COMPLETION","comments":[],"startOffset":144,"endOffset":149,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243601257+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"eb1dc583e51af761c5f1ce9024a3642d","type":"PII","value":"Umbrella Corporation","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 27/28: Redact contact Information (contains Producer)","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":128.8,"y":610.742},"width":106.16403,"height":-12.641998,"page":7}],"sectionNumber":27,"textBefore":"plant production Name: ","textAfter":" Contact: Jill","comments":[],"startOffset":208,"endOffset":228,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243602732+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":true,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"914e7b103972e36b06bab4d81dc0a60d","type":"purity","value":"99%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":164.8,"y":230.34204},"width":21.996002,"height":-12.642029,"page":8}],"sectionNumber":32,"textBefore":"Product-Code: EAK-L443 purity: ","textAfter":" ← not","comments":[],"startOffset":644,"endOffset":647,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243604682+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"161c4e3da3f995172904dc30843a60ac","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":8}],"sectionNumber":43,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243606629+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"b35fa03cf8dc88b6ecc1ddd1122d38cd","type":"purity","value":"101%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":164.8,"y":340.74203},"width":27.996002,"height":-12.641998,"page":8}],"sectionNumber":32,"textBefore":"be Hint Purity: ","textAfter":" ← should","comments":[],"startOffset":183,"endOffset":187,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243608065+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"fd877bb42040ac18eeea30d198f136b9","type":"purity","value":"45%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":178.408,"y":299.34204},"width":21.996002,"height":-12.642029,"page":8}],"sectionNumber":32,"textBefore":"symbols Purity: aa ","textAfter":" ← should","comments":[],"startOffset":374,"endOffset":377,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243609029+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"43284da7e4a2cbab327a3a46dd8ed5ec","type":"purity","value":"45%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":171.496,"y":368.34204},"width":21.996002,"height":-12.641998,"page":8}],"sectionNumber":32,"textBefore":"Hint Purity: <","textAfter":" ← should","comments":[],"startOffset":124,"endOffset":127,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243610032+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"810e2e001789f22ae3084a0a394dbc79","type":"purity","value":"45%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":164.8,"y":285.54205},"width":21.996002,"height":-12.642029,"page":8}],"sectionNumber":32,"textBefore":"additional symbols Purity: ","textAfter":" aa ←","comments":[],"startOffset":434,"endOffset":437,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243611061+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"7da27d812d47c2bdffb8e207b950a653","type":"purity","value":"45%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":171.496,"y":354.54202},"width":21.996002,"height":-12.641998,"page":8}],"sectionNumber":32,"textBefore":"Hint Purity: >","textAfter":" ← should","comments":[],"startOffset":154,"endOffset":157,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243612039+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"725b7fa856ec72811b053ec8ddb03da6","type":"purity","value":"45%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":178.288,"y":313.14203},"width":21.996002,"height":-12.641998,"page":8}],"sectionNumber":32,"textBefore":"symbols Purity: =<","textAfter":" ← should","comments":[],"startOffset":311,"endOffset":314,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243612999+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"d7f3c4658f09c65d17a1883fabd388dd","type":"purity","value":"99%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":171.496,"y":202.742},"width":21.996002,"height":-12.642029,"page":8}],"sectionNumber":32,"textBefore":"sensitive purity: <","textAfter":" ← not","comments":[],"startOffset":738,"endOffset":741,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243614289+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"fae4efb965c3c20715d7757c7956abbb","type":"purity","value":"99%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":171.496,"y":216.54205},"width":21.996002,"height":-12.642029,"page":8}],"sectionNumber":32,"textBefore":"sensitive purity: >","textAfter":" ← not","comments":[],"startOffset":691,"endOffset":694,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243615173+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"580aae7bbcd794b7b689aee1c82542b5","type":"purity","value":"45%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":164.8,"y":382.14203},"width":21.996002,"height":-12.641998,"page":8}],"sectionNumber":32,"textBefore":"Item: Soda Purity: ","textAfter":" ← should","comments":[],"startOffset":94,"endOffset":97,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243616142+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"1f9d675f57256b406d1a2baa3c201aff","type":"PII","value":"Umbrella Corporation","reason":"PERFORMING LABORATORY was found","matchedRule":20,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Rule 33/34: If Text contains \"Performing Lab\" and \"Lab Project ID\", then\nRedact everything between","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":236.8,"y":617.642},"width":106.16414,"height":-12.641998,"page":8}],"sectionNumber":30,"textBefore":"Report___ PERFORMING LABORATORY: ","textAfter":" LABORATORY PROJECT","comments":[],"startOffset":209,"endOffset":229,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243617175+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"5b889a101c0b5166aa0ea560b57d0d6b","type":"purity","value":"45%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":178.288,"y":326.94202},"width":21.996002,"height":-12.641998,"page":8}],"sectionNumber":32,"textBefore":"possible Purity: =>","textAfter":" ← should","comments":[],"startOffset":249,"endOffset":252,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243618051+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"d94bf1b70cfcb05ecb00994cc5abc890","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":8}],"sectionNumber":52,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.24361896+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"b35ccd2ec823eeae613bf62c1235b937","type":"purity","value":"45%","reason":"Purity found","matchedRule":17,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2a)","imported":false,"redacted":true,"section":"Rule 39: Purity Hint","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":164.8,"y":257.94202},"width":21.996002,"height":-12.642029,"page":8}],"sectionNumber":32,"textBefore":"additional symbols Purity: ","textAfter":"aa ← should","comments":[],"startOffset":559,"endOffset":562,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243619894+02:00"}],"manualChanges":[],"engines":["RULE"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":false,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"688b8e499afb958699834482b6c38323","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":734.742},"width":23.304,"height":-12.641998,"page":9}],"sectionNumber":44,"textBefore":"This is a ","textAfter":"-Header","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243620842+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"8fafcbefac14ac6b258d4ee07c7a0954","type":"PII","value":"Page","reason":"PII (Personal Identification Information) found","matchedRule":14,"rectangle":false,"legalBasis":"Reg (EC) No 1107/2009 Art. 63 (2e)","imported":false,"redacted":true,"section":"Footer","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.383995,"y":69.94202},"width":23.304,"height":-12.642029,"page":9}],"sectionNumber":53,"textBefore":"This is a ","textAfter":"-Footer","comments":[],"startOffset":10,"endOffset":14,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243621789+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"824ceb6b7d842de03ad899190fdae777","type":"CBI_author","value":"Müller","reason":"No Vertebrate found","matchedRule":2,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"Rule 41/42: Redact Signatures","color":[0.8,0.8,0.8],"positions":[{"topLeft":{"x":397.98404,"y":468.94202},"width":32.676025,"height":-12.641998,"page":9}],"sectionNumber":34,"textBefore":"Signed by: Tobias ","textAfter":null,"comments":[],"startOffset":204,"endOffset":210,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2023-05-25T12:24:33.243622799+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"image":false,"falsePositive":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index 548983ef..2f8890ef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -508,8 +508,8 @@ rule "102: Guidelines FileAttributes" when $section: Section(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS"))) then - RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.buildTextBlock()).stream() - .map(boundary -> $section.buildTextBlock().subSequence(boundary).toString()) + RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) .forEach(fileAttribute -> insert(fileAttribute)); end