From f81f7bd78825cb289cb7b742df4820a1552797a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Fri, 1 Sep 2023 13:50:47 +0200 Subject: [PATCH] RED-7317: fix behavior of recategorize --- .../service/ManualChangeFactory.java | 2 + .../v1/server/RedactionIntegrationTest.java | 1 + .../ManualChangesEnd2EndTest.java | 2 + .../ManualChangesIntegrationTest.java | 78 +++++++-- .../resources/drools/acceptance_rules.drl | 65 +++++++- .../src/test/resources/drools/all_rules.drl | 13 +- .../test/resources/drools/documine_flora.drl | 156 +++++++++++++++--- .../drools/manual_redaction_rules.drl | 15 +- .../src/test/resources/drools/rules.drl | 13 +- .../src/test/resources/drools/rules_v2.drl | 132 ++++++++++++++- 10 files changed, 418 insertions(+), 59 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java index b89409de..d93c881f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import java.time.OffsetDateTime; import java.util.List; import org.springframework.stereotype.Service; @@ -37,6 +38,7 @@ public class ManualChangeFactory { manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY) .withChange("value", manualRedactionEntry.getValue()); } + manualChange.setProcessedDate(OffsetDateTime.now()); return manualChange; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 83af03a9..b37e8d3d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -553,6 +553,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .fileId("fileId") .status(AnnotationStatus.APPROVED) .type("signature") + .requestDate(OffsetDateTime.now()) .build())); request.setManualRedactions(manualRedactions); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 4a5e073e..38009a3b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -169,6 +169,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .annotationId(idToResize) .value(expandedEntityKeyword) .positions(resizedPositions) + .requestDate(OffsetDateTime.now()) .status(AnnotationStatus.APPROVED) .build(); @@ -248,6 +249,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .fileId("fileId") .legalBasis("Manual Legal Basis Change") .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.now()) .build()))); analyzeService.reanalyze(request); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index 07732060..c6a83bab 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -7,6 +7,7 @@ import static org.wildfly.common.Assert.assertFalse; import java.awt.geom.Rectangle2D; import java.time.OffsetDateTime; import java.util.Collection; +import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -30,6 +31,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; @@ -40,6 +42,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; @Import(ManualChangesIntegrationTest.TestConfiguration.class) public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { @@ -49,7 +52,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; - private ManualChangesApplicationService manualChangesApplicationService; + private KieSession kieSession; @Qualifier("kieContainer") @Autowired @@ -79,8 +82,11 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { @BeforeEach public void createServices() { - entityCreationService = new EntityCreationService(entityEnrichmentService); - manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService); + kieSession = kieContainer.newKieSession(); + entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); + ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService); + kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); + kieSession.setGlobal("entityCreationService", entityCreationService); } @@ -101,13 +107,12 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { .value(biggerEntity.getValue()) .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0))) .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.now()) .build(); - KieSession kieSession = kieContainer.newKieSession(); - kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); + kieSession.insert(document); document.streamAllSubNodes().forEach(kieSession::insert); - kieSession.insert(entity); kieSession.insert(manualResizeRedaction); kieSession.fireAllRules(); kieSession.dispose(); @@ -139,9 +144,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { .requestDate(OffsetDateTime.now()) .build(); - KieSession kieSession = kieContainer.newKieSession(); - kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); - kieSession.insert(entity); + kieSession.insert(manualForceRedaction); kieSession.insert(document); document.streamAllSubNodes().forEach(kieSession::insert); @@ -171,11 +174,9 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build(); - KieSession kieSession = kieContainer.newKieSession(); - kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); + kieSession.insert(document); document.streamAllSubNodes().forEach(kieSession::insert); - kieSession.insert(entity); kieSession.insert(idRemoval); kieSession.fireAllRules(); kieSession.dispose(); @@ -203,11 +204,9 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { .requestDate(OffsetDateTime.now()) .build(); - KieSession kieSession = kieContainer.newKieSession(); - kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); + kieSession.insert(document); document.streamAllSubNodes().forEach(kieSession::insert); - kieSession.insert(entity); kieSession.insert(idRemoval); kieSession.insert(manualForceRedaction); kieSession.fireAllRules(); @@ -234,9 +233,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.REQUESTED).build(); - KieSession kieSession = kieContainer.newKieSession(); - kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); - kieSession.insert(entity); + kieSession.insert(idRemoval); kieSession.insert(document); document.streamAllSubNodes().forEach(kieSession::insert); @@ -253,6 +250,51 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { } + @Test + public void testManualRecategorizeRoundTrip() { + + Document document = buildGraph("files/new/crafted document"); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + + TextEntity entity = entities.stream() + .min(Comparator.comparingInt(textEntity -> textEntity.getTextRange().start())) + .orElseThrow(() -> new NotFoundException("no entity found!")); + String originalType = entity.getType(); + String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); + + OffsetDateTime start = OffsetDateTime.now(); + ManualImageRecategorization recategorization = ManualImageRecategorization.builder() + .type("any other type") + .annotationId(initialId) + .status(AnnotationStatus.APPROVED) + .requestDate(start) + .build(); + ManualImageRecategorization recategorization2 = ManualImageRecategorization.builder() + .type(originalType) + .annotationId(initialId) + .status(AnnotationStatus.APPROVED) + .requestDate(start.plusSeconds(1)) + .build(); + + + kieSession.insert(recategorization); + kieSession.insert(recategorization2); + kieSession.insert(document); + document.streamAllSubNodes().forEach(kieSession::insert); + kieSession.fireAllRules(); + kieSession.dispose(); + + var recategorizedEntity = document.getEntities() + .stream() + .filter(entity2 -> entity2.matchesAnnotationId(initialId)) + .findFirst() + .orElseThrow(() -> new NotFoundException("recategorized entity not found!")); + + assertEquals(originalType, recategorizedEntity.getManualOverwrite().getType().orElse(recategorizedEntity.getType())); + + } + + private void assertRectanglesAlmostEqual(Collection rects1, Collection rects2) { if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 39348dfd..e51f0b05 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -457,7 +457,6 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type == "dossier_redaction") then $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); - update($dossierRedaction); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end @@ -491,7 +490,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" rule "MAN.0.0: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); @@ -500,6 +500,19 @@ rule "MAN.0.0: Apply manual resize redaction" $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $imageToBeResized: Image(id == $id) + then + manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); + end + // Rule unit: MAN.1 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" @@ -540,12 +553,25 @@ rule "MAN.2.0: Apply force redaction" $entityToForce.getIntersectingNodes().forEach(node -> update(node)); end +rule "MAN.2.1: Apply force redaction to images" + no-loop true + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToForce: Image(id == $id) + then + $imageToForce.getManualOverwrite().addChange($force); + update($imageToForce); + update($imageToForce.getParent()); + end + // Rule unit: MAN.3 rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); @@ -555,6 +581,39 @@ rule "MAN.3.0: Apply entity recategorization" retract($entityToBeRecategorized); end +rule "MAN.3.1: Apply image recategorization" + salience 128 + when + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $imageToBeRecategorized: Image($id == id) + then + manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); + update($imageToBeRecategorized); + update($imageToBeRecategorized.getParent()); + retract($recategorization); + end + + +// Rule unit: MAN.4 +rule "MAN.4.0: Apply legal basis change" + salience 128 + when + $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToBeRecategorized: Image($id == id) + then + $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + end + +rule "MAN.4.1: Apply legal basis change" + salience 128 + when + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeChanged: TextEntity(matchesAnnotationId($id)) + then + $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + end + //------------------------------------ Entity merging rules ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index 0108d8d3..13438536 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -1073,7 +1073,6 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type == "dossier_redaction") then $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); - update($dossierRedaction); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end @@ -1162,7 +1161,8 @@ rule "AI.2.0: add all NER Entities of any type except CBI_author" rule "MAN.0.0: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); @@ -1174,7 +1174,8 @@ rule "MAN.0.0: Apply manual resize redaction" rule "MAN.0.1: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $imageToBeResized: Image(id == $id) then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); @@ -1240,7 +1241,8 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); @@ -1253,7 +1255,8 @@ rule "MAN.3.0: Apply entity recategorization" rule "MAN.3.1: Apply image recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index bdd925ef..091a168c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1261,12 +1261,27 @@ rule "DOC.45.0: Doses (mg/kg bodyweight)" rule "MAN.0.0: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); + $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $imageToBeResized: Image(id == $id) + then + manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); end @@ -1274,50 +1289,99 @@ rule "MAN.0.0: Apply manual resize redaction" rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) - not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.removeFromGraph(); - retract($entityToBeRemoved); + $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) - not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) $imageEntityToBeRemoved: Image($id == id) then - $imageEntityToBeRemoved.setIgnored(true); + $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); end // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" + no-loop true salience 128 when - $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); - $entityToForce.setRemoved(false); - $entityToForce.setIgnored(false); - $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + $entityToForce.getManualOverwrite().addChange($force); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); - retract($force); + end + +rule "MAN.2.1: Apply force redaction to images" + no-loop true + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToForce: Image(id == $id) + then + $imageToForce.getManualOverwrite().addChange($force); + update($imageToForce); + update($imageToForce.getParent()); end // Rule unit: MAN.3 -rule "MAN.3.0: Apply image recategorization" +rule "MAN.3.0: Apply entity recategorization" salience 128 when - ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) - $image: Image($id == id) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then - $image.setImageType(ImageType.fromString($imageType)); + $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); + manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + retract($recategorization); + // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. + retract($entityToBeRecategorized); + end + +rule "MAN.3.1: Apply image recategorization" + salience 128 + when + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $imageToBeRecategorized: Image($id == id) + then + manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); + update($imageToBeRecategorized); + update($imageToBeRecategorized.getParent()); + retract($recategorization); + end + +// Rule unit: MAN.4 +rule "MAN.4.0: Apply legal basis change" + salience 128 + when + $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToBeRecategorized: Image($id == id) + then + $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + end + +rule "MAN.4.1: Apply legal basis change" + salience 128 + when + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeChanged: TextEntity(matchesAnnotationId($id)) + then + $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); end @@ -1328,14 +1392,64 @@ rule "MAN.3.0: Apply image recategorization" rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, getActive()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, getActive()) + $larger: TextEntity($type: type, $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end +// Rule unit: X.2 +rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 64 + when + $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); + retract($entity) + end + + +// Rule unit: X.3 +rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 64 + when + $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + then + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); + retract($recommendation); + end + + +// Rule unit: X.4 +rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" + salience 256 + when + $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) + $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + then + $entity.addEngines($recommendation.getEngines()); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); + retract($recommendation); + end + + +// Rule unit: X.5 +rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 256 + when + $entity: TextEntity(entityType == EntityType.ENTITY, active()) + $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active()) + then + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); + retract($recommendation); + end + + // Rule unit: X.7 rule "X.7.0: remove all images" salience 512 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index c748f8ae..a89c2384 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -65,7 +65,8 @@ query "getFileAttributes" rule "MAN.0.0: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); @@ -77,7 +78,8 @@ rule "MAN.0.0: Apply manual resize redaction" rule "MAN.0.1: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $imageToBeResized: Image(id == $id) then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); @@ -143,11 +145,12 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); + manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); retract($recategorization); // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. retract($entityToBeRecategorized); @@ -156,7 +159,8 @@ rule "MAN.3.0: Apply entity recategorization" rule "MAN.3.1: Apply image recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); @@ -184,7 +188,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); end - //------------------------------------ Local dictionary search rules ------------------------------------ // Rule unit: LDS.0 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 7174041b..58cdc38a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -798,7 +798,6 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type == "dossier_redaction") then $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); - update($dossierRedaction); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end @@ -875,7 +874,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" rule "MAN.0.0: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); @@ -887,7 +887,8 @@ rule "MAN.0.0: Apply manual resize redaction" rule "MAN.0.1: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $imageToBeResized: Image(id == $id) then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); @@ -953,7 +954,8 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); @@ -966,7 +968,8 @@ rule "MAN.3.0: Apply entity recategorization" rule "MAN.3.1: Apply image recategorization" salience 128 when - $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index ce547025..2296d496 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -33,6 +33,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; @@ -89,6 +90,136 @@ rule "Always redact PII" end +//------------------------------------ Manual redaction rules ------------------------------------ + +// Rule unit: MAN.0 +rule "MAN.0.0: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $entityToBeResized: TextEntity(matchesAnnotationId($id)) + then + manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($entityToBeResized); + $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $imageToBeResized: Image(id == $id) + then + manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); + end + + +// Rule unit: MAN.1 +rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" + salience 128 + when + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) + then + $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" + salience 128 + when + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) + $imageEntityToBeRemoved: Image($id == id) + then + $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); + end + + +// Rule unit: MAN.2 +rule "MAN.2.0: Apply force redaction" + no-loop true + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToForce: TextEntity(matchesAnnotationId($id)) + then + $entityToForce.getManualOverwrite().addChange($force); + update($entityToForce); + $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.2.1: Apply force redaction to images" + no-loop true + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToForce: Image(id == $id) + then + $imageToForce.getManualOverwrite().addChange($force); + update($imageToForce); + update($imageToForce.getParent()); + end + + +// Rule unit: MAN.3 +rule "MAN.3.0: Apply entity recategorization" + salience 128 + when + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) + then + $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); + manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + retract($recategorization); + // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. + retract($entityToBeRecategorized); + end + +rule "MAN.3.1: Apply image recategorization" + salience 128 + when + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $imageToBeRecategorized: Image($id == id) + then + manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); + update($imageToBeRecategorized); + update($imageToBeRecategorized.getParent()); + retract($recategorization); + end + + +// Rule unit: MAN.4 +rule "MAN.4.0: Apply legal basis change" + salience 128 + when + $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToBeRecategorized: Image($id == id) + then + $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + end + +rule "MAN.4.1: Apply legal basis change" + salience 128 + when + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeChanged: TextEntity(matchesAnnotationId($id)) + then + $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + end + //------------------------------------ Entity merging rules ------------------------------------ @@ -211,4 +342,3 @@ rule "LDS.0.0: run local dictionary search" entity.addMatchedRules(matchedRules); }); end -