From b9c9d764421c8c3bd7e774bf46daa5b45c3bfe31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Mon, 4 Sep 2023 09:34:53 +0200 Subject: [PATCH] RED-7317: fix behavior of recategorize --- .../build.gradle.kts | 2 +- .../graph/entity/ManualChangeOverwrite.java | 6 +-- .../services/EntityCreationService.java | 40 +++++++++---------- .../ManualChangesApplicationService.java | 10 ++--- .../service/DroolsExecutionService.java | 33 ++------------- .../service/ManualChangeFactory.java | 4 +- .../service/ManualRedactionEntryService.java | 2 +- .../service/SectionFinderService.java | 4 +- .../v1/server/RedactionIntegrationTest.java | 4 +- ...ocumentEntityInsertionIntegrationTest.java | 4 +- .../ManualChangesEnd2EndTest.java | 6 +-- .../ManualChangesIntegrationTest.java | 6 +-- .../manualchanges/ManualChangesUnitTest.java | 4 +- .../adapter/NerEntitiesAdapterTest.java | 6 +-- .../resources/drools/acceptance_rules.drl | 31 +++++++++----- .../src/test/resources/drools/all_rules.drl | 26 ++++++------ .../test/resources/drools/documine_flora.drl | 10 ++--- .../drools/manual_redaction_rules.drl | 10 ++--- .../src/test/resources/drools/rules.drl | 29 +++++++++----- .../src/test/resources/drools/rules_v2.drl | 12 +++--- .../EFSA_sanitisation_GFL_v1/rules.drl | 8 ++-- 21 files changed, 127 insertions(+), 130 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index e3c1b5a7..cbed98ec 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.25.0" val jacksonVersion = "2.15.2" val droolsVersion = "8.43.0.Final" val pdfBoxVersion = "3.0.0-alpha2" -val persistenceServiceVersion = "2.155.0" +val persistenceServiceVersion = "2.156.0" configurations { all { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ManualChangeOverwrite.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ManualChangeOverwrite.java index d7c267eb..e28fda4c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ManualChangeOverwrite.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ManualChangeOverwrite.java @@ -10,7 +10,7 @@ import java.util.Optional; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; @@ -33,7 +33,7 @@ public class ManualChangeOverwrite { ManualResizeRedaction.class, "resized by manual override", // ManualForceRedaction.class, "forced by manual override", // IdRemoval.class, "removed by manual override", // - ManualImageRecategorization.class, "recategorized by manual override"); + ManualRecategorization.class, "recategorized by manual override"); List manualChanges = new LinkedList<>(); boolean changed; @@ -101,7 +101,7 @@ public class ManualChangeOverwrite { resized = true; } - if (manualChange instanceof ManualImageRecategorization recategorization) { + if (manualChange instanceof ManualRecategorization recategorization) { // recategorization logic happens in ManualChangesApplicationService. recategorized = true; // this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService. diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java index c048d2ab..a39ae9b8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java @@ -195,7 +195,7 @@ public class EntityCreationService { return entityBoundaries.stream() .map(boundary -> boundary.trim(node.getTextBlock())) .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -236,7 +236,7 @@ public class EntityCreationService { return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) .stream() .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) - .map(bounds -> byBoundary(bounds, type, entityType, node)) + .map(bounds -> byTextRange(bounds, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -250,7 +250,7 @@ public class EntityCreationService { .stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -264,7 +264,7 @@ public class EntityCreationService { .stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -277,7 +277,7 @@ public class EntityCreationService { .stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -290,7 +290,7 @@ public class EntityCreationService { .stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -338,7 +338,7 @@ public class EntityCreationService { .map(nextTableCell -> RedactionSearchUtility.findBoundaryOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock())) .map(b -> b.trim(tableNode.getTextBlock())) .filter(boundary -> isValidEntityBoundary(tableNode.getTextBlock(), boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, tableNode)) + .map(boundary -> byTextRange(boundary, type, entityType, tableNode)) .filter(Optional::isPresent) .map(Optional::get)) .flatMap(Functions.identity()); @@ -356,7 +356,7 @@ public class EntityCreationService { if (!isValidEntityBoundary(textBlock, boundary)) { return Optional.empty(); } - return byBoundary(boundary, type, entityType, semanticNode); + return byTextRange(boundary, type, entityType, semanticNode); } @@ -388,7 +388,7 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -398,7 +398,7 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -408,7 +408,7 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -418,7 +418,7 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexPattern, group, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -428,7 +428,7 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -438,7 +438,7 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByStringIgnoreCase(keyword, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -456,7 +456,7 @@ public class EntityCreationService { .map(SemanticNode::getTextRange) .collect(new ConsecutiveBoundaryCollector()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)) + .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -468,7 +468,7 @@ public class EntityCreationService { return Optional.empty(); } TextRange textRange = new TextRange(node.getTextBlock().indexOf(string) + string.length(), node.getTextRange().end()); - return byBoundary(textRange, type, entityType, node); + return byTextRange(textRange, type, entityType, node); } @@ -482,14 +482,14 @@ public class EntityCreationService { if (!isValidEntityBoundary(node.getTextBlock(), textRange)) { return Optional.empty(); } - return byBoundary(textRange, type, entityType, node); + return byTextRange(textRange, type, entityType, node); } public Optional byPrefixExpansionRegex(TextEntity entity, String regexPattern) { int expandedStart = RedactionSearchUtility.getExpandedStartByRegex(entity, regexPattern); - return byBoundary(new TextRange(expandedStart, entity.getTextRange().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); + return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); } @@ -497,7 +497,7 @@ public class EntityCreationService { int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern); expandedEnd = truncateEndIfLineBreakIsBetween(entity.getTextRange().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock()); - return byBoundary(new TextRange(entity.getTextRange().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); + return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); } @@ -521,7 +521,7 @@ public class EntityCreationService { * @param node The semantic node to associate with the redaction entity. * @return An Optional containing the redaction entity, or the previous entity if the entity already exists. */ - public Optional byBoundary(TextRange textRange, String type, EntityType entityType, SemanticNode node) { + public Optional byTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) { if (!node.getTextRange().contains(textRange)) { throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualChangesApplicationService.java index ee701a12..74836290 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualChangesApplicationService.java @@ -5,7 +5,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.NoSuchElementException; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.document.graph.entity.Entity; import com.iqser.red.service.redaction.v1.server.document.graph.entity.PositionOnPage; @@ -23,17 +23,17 @@ public class ManualChangesApplicationService { private final EntityCreationService entityCreationService; - public void recategorize(Entity entityToBeReCategorized, ManualImageRecategorization manualImageRecategorization) { + public void recategorize(Entity entityToBeReCategorized, ManualRecategorization manualRecategorization) { if (entityToBeReCategorized instanceof Image image) { - image.setImageType(ImageType.fromString(manualImageRecategorization.getType())); + image.setImageType(ImageType.fromString(manualRecategorization.getType())); return; } // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed! if (entityToBeReCategorized instanceof TextEntity textEntity) { - TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualImageRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode()); + TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode()); recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage()); - recategorizedEntity.getManualOverwrite().addChange(manualImageRecategorization); + recategorizedEntity.getManualOverwrite().addChange(manualRecategorization); textEntity.removeFromGraph(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index d58ab3dd..d17761ee 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -1,18 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toList; - import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.Comparator; import java.util.LinkedList; import java.util.List; -import java.util.stream.Collector; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.kie.api.KieServices; @@ -27,9 +19,6 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; @@ -100,9 +89,10 @@ public class DroolsExecutionService { if (manualRedactions != null) { manualRedactions.getResizeRedactions().forEach(kieSession::insert); - manualRedactions.getImageRecategorization().forEach(kieSession::insert); + manualRedactions.getRecategorizations().forEach(kieSession::insert); manualRedactions.getEntriesToAdd().forEach(kieSession::insert); - insertLatestForceOrRemovalPerAnnotationId(manualRedactions.getForceRedactions(), manualRedactions.getIdsToRemove(), kieSession); + manualRedactions.getForceRedactions().forEach(kieSession::insert); + manualRedactions.getIdsToRemove().forEach(kieSession::insert); } kieSession.insert(nerEntities); @@ -115,23 +105,6 @@ public class DroolsExecutionService { } - private static void insertLatestForceOrRemovalPerAnnotationId(Collection forceRedactions, Collection idRemovals, KieSession kieSession) { - - Stream.concat(forceRedactions.stream(), idRemovals.stream()) - .filter(BaseAnnotation::isApproved) - .filter(baseAnnotation -> baseAnnotation.getRequestDate() != null) - .collect(groupingBy(BaseAnnotation::getAnnotationId, sortByRequestDate())) - .values() - .forEach(baseAnnotations -> kieSession.insert(baseAnnotations.get(0))); - } - - - private static Collector> sortByRequestDate() { - - return collectingAndThen(toList(), originList -> originList.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate).reversed()).toList()); - } - - public List getFileAttributes(KieSession kieSession) { List fileAttributes = new LinkedList<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java index d93c881f..ba0b3950 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java @@ -8,7 +8,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; @@ -26,7 +26,7 @@ public class ManualChangeFactory { private ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) { ManualChange manualChange = ManualChange.from(baseAnnotation); - if (baseAnnotation instanceof ManualImageRecategorization imageRecategorization) { + if (baseAnnotation instanceof ManualRecategorization imageRecategorization) { manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()); } else if (baseAnnotation instanceof IdRemoval manualRemoval) { manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionEntryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionEntryService.java index 632ce8f5..1dc59c7e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionEntryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionEntryService.java @@ -50,7 +50,7 @@ public class ManualRedactionEntryService { return Stream.of(manualRedactions.getForceRedactions(), manualRedactions.getResizeRedactions(), - manualRedactions.getImageRecategorization(), + manualRedactions.getRecategorizations(), manualRedactions.getIdsToRemove(), manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java index 75ee665b..067a0229 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java @@ -15,8 +15,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; @@ -107,7 +107,7 @@ class SectionFinderService { return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId), Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId), - Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId), + Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index b37e8d3d..9db62950 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -49,7 +49,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; @@ -548,7 +548,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ManualRedactions manualRedactions = new ManualRedactions(); - manualRedactions.setImageRecategorization(Set.of(ManualImageRecategorization.builder() + manualRedactions.setRecategorizations(Set.of(ManualRecategorization.builder() .annotationId("37eee3e9d589a5cc529bfec38c3ba479") .fileId("fileId") .status(AnnotationStatus.APPROVED) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java index bc782b13..d35cc3c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java @@ -67,8 +67,8 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra Document document = buildGraph("files/new/crafted document.pdf"); String type = "CBI_author"; - assertTrue(entityCreationService.byBoundary(new TextRange(0, 10), type, EntityType.ENTITY, document).isPresent()); - assertTrue(entityCreationService.byBoundary(new TextRange(0, 10), type, EntityType.ENTITY, document).isPresent()); + assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document).isPresent()); + assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document).isPresent()); assertEquals(1, document.getEntities().size()); verify(kieSession, times(1)).insert(any(TextEntity.class)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 38009a3b..2977e332 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -39,7 +39,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; @@ -294,7 +294,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { assertEquals("CBI.3.2", asyaLyon.getMatchedRule()); assertEquals("No vertebrate found", asyaLyon.getReason()); - ManualImageRecategorization recategorization = ManualImageRecategorization.builder() + ManualRecategorization recategorization = ManualRecategorization.builder() .requestDate(OffsetDateTime.now()) .status(AnnotationStatus.APPROVED) .type("vertebrate") @@ -303,7 +303,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .build(); request.setManualRedactions(new ManualRedactions()); - request.getManualRedactions().setImageRecategorization(Set.of(recategorization)); + request.getManualRedactions().setRecategorizations(Set.of(recategorization)); analyzeService.reanalyze(request); RedactionLog redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index c6a83bab..6b6f04c9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -31,7 +31,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; @@ -263,13 +263,13 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); OffsetDateTime start = OffsetDateTime.now(); - ManualImageRecategorization recategorization = ManualImageRecategorization.builder() + ManualRecategorization recategorization = ManualRecategorization.builder() .type("any other type") .annotationId(initialId) .status(AnnotationStatus.APPROVED) .requestDate(start) .build(); - ManualImageRecategorization recategorization2 = ManualImageRecategorization.builder() + ManualRecategorization recategorization2 = ManualRecategorization.builder() .type(originalType) .annotationId(initialId) .status(AnnotationStatus.APPROVED) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index 00b71cee..b5f8e0c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -14,7 +14,7 @@ import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; @@ -119,7 +119,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())); - ManualImageRecategorization imageRecategorizationRequest = ManualImageRecategorization.builder() + ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder() .type("type") .requestDate(start.plusSeconds(5)) .annotationId(annotationId) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index 2fb4cd37..95ca0e3b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -76,7 +76,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { .stream() .filter(e -> !e.type().equals("CBI_author")); List redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts) - .map(e -> entityCreationService.byBoundary(e.textRange(), e.type(), EntityType.ENTITY, document)) + .map(e -> entityCreationService.byTextRange(e.textRange(), e.type(), EntityType.ENTITY, document)) .filter(Optional::isPresent) .map(Optional::get) .toList(); @@ -110,7 +110,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { List nerEntityBoundaries = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).toList(); log.info("Combined to CBI_address"); List cbiAddressEntities = nerEntityBoundaries.stream() - .map(b -> entityCreationService.byBoundary(b, "CBI_address", EntityType.RECOMMENDATION, document)) + .map(b -> entityCreationService.byTextRange(b, "CBI_address", EntityType.RECOMMENDATION, document)) .filter(Optional::isPresent) .map(Optional::get) .toList(); @@ -123,7 +123,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { List validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) .getNerEntityList() .stream() - .map(e -> entityCreationService.byBoundary(e.textRange(), e.type(), EntityType.ENTITY, document)) + .map(e -> entityCreationService.byTextRange(e.textRange(), e.type(), EntityType.ENTITY, document)) .filter(Optional::isPresent) .map(Optional::get) .toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index e51f0b05..34d47b20 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dict import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; @@ -115,7 +115,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author" when $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied()) then - entityCreationService.byBoundary($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) + entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); end @@ -542,7 +542,6 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" - no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) @@ -551,10 +550,10 @@ rule "MAN.2.0: Apply force redaction" $entityToForce.getManualOverwrite().addChange($force); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); end rule "MAN.2.1: Apply force redaction to images" - no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) @@ -563,6 +562,7 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce.getManualOverwrite().addChange($force); update($imageToForce); update($imageToForce.getParent()); + retract($force); end @@ -570,9 +570,9 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); @@ -581,11 +581,22 @@ rule "MAN.3.0: Apply entity recategorization" retract($entityToBeRecategorized); end -rule "MAN.3.1: Apply image recategorization" +rule "MAN.3.1: Apply entity recategorization of same type" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + then + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + retract($recategorization); + end + +rule "MAN.3.2: Apply image recategorization" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index 13438536..0de1d713 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dict import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; @@ -128,7 +128,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author" when $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied()) then - entityCreationService.byBoundary($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) + entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); end @@ -1213,7 +1213,6 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" - no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) @@ -1222,10 +1221,10 @@ rule "MAN.2.0: Apply force redaction" $entityToForce.getManualOverwrite().addChange($force); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); end rule "MAN.2.1: Apply force redaction to images" - no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) @@ -1234,6 +1233,7 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce.getManualOverwrite().addChange($force); update($imageToForce); update($imageToForce.getParent()); + retract($force); end @@ -1241,22 +1241,24 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + if (!$recategorization.getType().equals($entityToBeRecategorized.getType())) { + $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); + manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. + retract($entityToBeRecategorized); + } retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply image recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 091a168c..4f79d270 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dict import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; @@ -1341,8 +1341,8 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); @@ -1355,8 +1355,8 @@ rule "MAN.3.0: Apply entity recategorization" rule "MAN.3.1: Apply image recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index a89c2384..882f0579 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dict import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; @@ -145,8 +145,8 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); @@ -159,8 +159,8 @@ rule "MAN.3.0: Apply entity recategorization" rule "MAN.3.1: Apply image recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 58cdc38a..0e6dc251 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dict import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; @@ -926,7 +926,6 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" - no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) @@ -935,10 +934,10 @@ rule "MAN.2.0: Apply force redaction" $entityToForce.getManualOverwrite().addChange($force); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); end rule "MAN.2.1: Apply force redaction to images" - no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) @@ -947,6 +946,7 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce.getManualOverwrite().addChange($force); update($imageToForce); update($imageToForce.getParent()); + retract($force); end @@ -954,9 +954,9 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); @@ -965,11 +965,22 @@ rule "MAN.3.0: Apply entity recategorization" retract($entityToBeRecategorized); end -rule "MAN.3.1: Apply image recategorization" +rule "MAN.3.1: Apply entity recategorization of same type" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + then + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + retract($recategorization); + end + +rule "MAN.3.2: Apply image recategorization" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 2296d496..c32bcb56 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dict import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; @@ -65,7 +65,7 @@ rule "add NER Entities of type CBI_author or CBI_address" when $nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address")) then - entityCreationService.byBoundary(new TextRange($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document) + entityCreationService.byTextRange(new TextRange($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document) .ifPresent(redactionEntity -> insert(redactionEntity)); end @@ -176,8 +176,8 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); @@ -190,8 +190,8 @@ rule "MAN.3.0: Apply entity recategorization" rule "MAN.3.1: Apply image recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) - not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index ab9db674..e8f074d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dict import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; @@ -115,7 +115,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author" when $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied()) then - entityCreationService.byBoundary($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) + entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); end @@ -571,7 +571,7 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); @@ -584,7 +584,7 @@ rule "MAN.3.0: Apply entity recategorization" rule "MAN.3.1: Apply image recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);