diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java similarity index 91% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java index 84913f06..48bca5a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java @@ -12,7 +12,7 @@ import lombok.NoArgsConstructor; @CompiledJson @AllArgsConstructor @NoArgsConstructor -public class EntityRecogintionEntity { +public class EntityRecognitionEntity { private String value; private int startOffset; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java index 466cc3aa..71522a8a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java @@ -16,6 +16,6 @@ import lombok.NoArgsConstructor; @AllArgsConstructor public class NerEntities { - private Map> data = new HashMap<>(); + private Map> data = new HashMap<>(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java index e3a824ee..92ebe717 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java @@ -161,9 +161,9 @@ public class EntityNode { } - public boolean matchesResizedAnnotationId(String resizedAnnotationId) { + public boolean matchesAnnotationId(String manualRedactionId) { - return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(resizedAnnotationId)); + return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java index e52bf261..5cbc27bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java @@ -4,6 +4,7 @@ import static java.lang.String.format; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; +import java.awt.geom.Rectangle2D; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -37,6 +38,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; +import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; @@ -254,10 +256,12 @@ public class DocumentGraphFactory { private void addImage(SectionNode sectionNode, PdfImage image, Context context) { + Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition()); PageNode page = getPage(image.getPage(), context); ImageNode imageNode = ImageNode.builder() + .id(IdBuilder.buildId(Set.of(page), List.of(position))) .imageType(image.getImageType()) - .position(RectangleTransformations.toRectangle2D(image.getPosition())) + .position(position) .transparency(image.isHasTransparency()) .page(page) .tableOfContents(context.tableOfContents()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java index aeb94dc4..240f15d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java @@ -112,6 +112,12 @@ public class RectangleTransformations { } + public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) { + + return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight()); + } + + private static class Rectangle2DUnion implements Collector { @Override diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java index 11a42505..4fcaa5e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java @@ -30,11 +30,12 @@ import lombok.experimental.FieldDefaults; public class ImageNode implements SemanticNode { List tocId; + String id; ImageType imageType; boolean transparency; Rectangle2D position; - + boolean redaction; boolean ignored; @Builder.Default diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java index 4dfaa40d..0582c581 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java @@ -19,6 +19,7 @@ public class PropertiesMapper { properties.put("imageType", image.getImageType().toString()); properties.put("transparency", String.valueOf(image.isTransparency())); properties.put("position", RectangleTransformations.toString(image.getPosition())); + properties.put("id", image.getId()); return properties; } @@ -54,6 +55,7 @@ public class PropertiesMapper { builder.imageType(parseImageType(properties.get("imageType"))); builder.transparency(Boolean.parseBoolean(properties.get("transparency"))); builder.position(parseRectangle2D(properties.get("position"))); + builder.id(properties.get("id")); } @@ -73,7 +75,7 @@ public class PropertiesMapper { } - private static ImageType parseImageType(String imageType) { + public static ImageType parseImageType(String imageType) { return switch (imageType) { case "LOGO" -> ImageType.LOGO; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualRedactionApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualRedactionApplicationService.java new file mode 100644 index 00000000..646d1536 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualRedactionApplicationService.java @@ -0,0 +1,45 @@ +package com.iqser.red.service.redaction.v1.server.document.services; + +import java.util.NoSuchElementException; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition; +import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class ManualRedactionApplicationService { + + private final EntityCreationService entityCreationService; + + + public void resizeEntityAndReinsert(EntityNode entityToBeResized, ManualResizeRedaction manualResizeRedaction) { + + RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage() + .stream() + .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getValue())) + .findFirst() + .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!")); + + redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(RectangleTransformations::toRectangle2D).toList()); + + int newStartOffset; + if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) { + newStartOffset = entityToBeResized.getBoundary().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue()); + } else { + newStartOffset = entityToBeResized.getBoundary().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue()); + } + + entityToBeResized.setResized(true); + entityToBeResized.getBoundary().setStart(newStartOffset); + entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length()); + entityToBeResized.removeFromGraph(); + entityCreationService.addEntityToGraph(entityToBeResized, entityToBeResized.getDeepestFullyContainingNode().getTableOfContents()); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index dac47463..bb5ef495 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -3,6 +3,7 @@ package drools import static java.lang.String.format; import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch; import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields; +import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseImageType; import java.util.List; @@ -21,9 +22,15 @@ import com.iqser.red.service.redaction.v1.server.document.services.EntityCreatio import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; +import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService; global DocumentGraph document global EntityCreationService entityCreationService +global ManualRedactionApplicationService manualRedactionApplicationService global Dictionary dictionary // --------------------------------------- queries ------------------------------------------------------------------- @@ -32,6 +39,60 @@ query "getFileAttributes" $fileAttribute: FileAttribute() end +// --------------------------------------- manual redaction rules ------------------------------------------------------------------- + +rule "Apply manual resize redaction" + salience 200 + when + $resizeRedactions: ManualResizeRedaction($id: annotationId) + $entityToBeResized: EntityNode(matchesAnnotationId($id)) + then + manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedactions); + retract($resizeRedactions); + update($entityToBeResized); + end + +rule "Apply id removals that are valid and not in forced redactions to Entity" + salience 200 + when + IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $entityToBeRemoved: EntityNode(matchesAnnotationId($id)) + then + $entityToBeRemoved.removeFromGraph(); + retract($entityToBeRemoved); + end + +rule "Apply id removals that are valid and not in forced redactions to Image" + salience 200 + when + IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $entityToBeRemoved: ImageNode($id == id) + then + $entityToBeRemoved.setIgnored(true); + end + +rule "Apply force redaction" + salience 200 + when + ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $entityToForce: EntityNode(matchesAnnotationId($id)) + then + $entityToForce.setLegalBasis($legalBasis); + $entityToForce.setRedaction(true); + $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + end + +rule "Apply image recategorization" + salience 200 + when + ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) + $image: ImageNode($id == id) + then + $image.setImageType(parseImageType($imageType)); + end + // --------------------------------------- merging rules ------------------------------------------------------------------- rule "merge intersecting Entities of same type" @@ -107,7 +168,6 @@ rule "run local dictionary search" entityNodes.forEach(entityNode -> insert(entityNode)); end - // --------------------------------------- CBI rules ------------------------------------------------------------------- rule "1: Redact CBI Authors (Non vertebrate study)" @@ -167,7 +227,6 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" insert(falsePositive); end - rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation" agenda-group "LOCAL_DICTIONARY_ADDS" @@ -203,7 +262,6 @@ rule "8: Add recommendation for Addresses in Test Organism sections" entities.forEach(entity -> insert(entity)); end - rule "9: Add recommendation for Addresses in Test Animals sections" when @@ -226,7 +284,6 @@ rule "12: Recommend CTL/BL laboratory that start with BL or CTL" entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end - // --------------------------------------- PII rules ------------------------------------------------------------------- @@ -242,8 +299,6 @@ rule "10: Redacted PII Personal Identification Information (Non vertebrate study update($entity) end - - rule "11: Redacted PII Personal Identification Information (Vertebrate study)" no-loop true @@ -278,7 +333,6 @@ rule "13: Redact Emails by RegEx (Vertebrate study)" entities.forEach(entity -> insert(entity)); end - rule "14: Redact line after contact information (Non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" @@ -313,7 +367,6 @@ rule "14: Redact line after contact information (Non vertebrate study)" entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end - rule "15: Redact line after contact information (Vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" @@ -346,7 +399,6 @@ rule "15: Redact line after contact information (Vertebrate study)" entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end - rule "16: redact line between contact keywords" agenda-group "LOCAL_DICTIONARY_ADDS"