Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7317' into 'master'

Browse Source 
RED-7317: fix behavior of recategorize

Closes RED-7317

See merge request redactmanager/redaction-service!115
This commit is contained in:
Kilian Schüttler 2023-09-01 13:50:47 +02:00
commit 08662410f7
10 changed files with 418 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.time.OffsetDateTime;
import java.util.List;
import org.springframework.stereotype.Service;
@ -37,6 +38,7 @@ public class ManualChangeFactory {
manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)
.withChange("value", manualRedactionEntry.getValue());
}
manualChange.setProcessedDate(OffsetDateTime.now());
return manualChange;
}

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -553,6 +553,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.fileId("fileId")
.status(AnnotationStatus.APPROVED)
.type("signature")
.requestDate(OffsetDateTime.now())
.build()));
request.setManualRedactions(manualRedactions);

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -169,6 +169,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.annotationId(idToResize)
.value(expandedEntityKeyword)
.positions(resizedPositions)
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.build();
@ -248,6 +249,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.fileId("fileId")
.legalBasis("Manual Legal Basis Change")
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.now())
.build())));
analyzeService.reanalyze(request);

78
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java
View File

@ -7,6 +7,7 @@ import static org.wildfly.common.Assert.assertFalse;
import java.awt.geom.Rectangle2D;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@ -30,6 +31,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType;
@ -40,6 +42,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
@Import(ManualChangesIntegrationTest.TestConfiguration.class)
public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
@ -49,7 +52,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
private EntityCreationService entityCreationService;
private ManualChangesApplicationService manualChangesApplicationService;
private KieSession kieSession;
@Qualifier("kieContainer")
@Autowired
@ -79,8 +82,11 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
@BeforeEach
public void createServices() {
entityCreationService = new EntityCreationService(entityEnrichmentService);
manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
kieSession = kieContainer.newKieSession();
entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.setGlobal("entityCreationService", entityCreationService);
}
@ -101,13 +107,12 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
.value(biggerEntity.getValue())
.positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0)))
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.now())
.build();
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.insert(entity);
kieSession.insert(manualResizeRedaction);
kieSession.fireAllRules();
kieSession.dispose();
@ -139,9 +144,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
.requestDate(OffsetDateTime.now())
.build();
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.insert(entity);
kieSession.insert(manualForceRedaction);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
@ -171,11 +174,9 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build();
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.insert(entity);
kieSession.insert(idRemoval);
kieSession.fireAllRules();
kieSession.dispose();
@ -203,11 +204,9 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
.requestDate(OffsetDateTime.now())
.build();
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.insert(entity);
kieSession.insert(idRemoval);
kieSession.insert(manualForceRedaction);
kieSession.fireAllRules();
@ -234,9 +233,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.REQUESTED).build();
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.insert(entity);
kieSession.insert(idRemoval);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
@ -253,6 +250,51 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
}
@Test
public void testManualRecategorizeRoundTrip() {
Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream()
.min(Comparator.comparingInt(textEntity -> textEntity.getTextRange().start()))
.orElseThrow(() -> new NotFoundException("no entity found!"));
String originalType = entity.getType();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
OffsetDateTime start = OffsetDateTime.now();
ManualImageRecategorization recategorization = ManualImageRecategorization.builder()
.type("any other type")
.annotationId(initialId)
.status(AnnotationStatus.APPROVED)
.requestDate(start)
.build();
ManualImageRecategorization recategorization2 = ManualImageRecategorization.builder()
.type(originalType)
.annotationId(initialId)
.status(AnnotationStatus.APPROVED)
.requestDate(start.plusSeconds(1))
.build();
kieSession.insert(recategorization);
kieSession.insert(recategorization2);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.fireAllRules();
kieSession.dispose();
var recategorizedEntity = document.getEntities()
.stream()
.filter(entity2 -> entity2.matchesAnnotationId(initialId))
.findFirst()
.orElseThrow(() -> new NotFoundException("recategorized entity not found!"));
assertEquals(originalType, recategorizedEntity.getManualOverwrite().getType().orElse(recategorizedEntity.getType()));
}
private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {

65
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -457,7 +457,6 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
$dossierRedaction: TextEntity(type == "dossier_redaction")
then
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
update($dossierRedaction);
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
end
@ -491,7 +490,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
rule "MAN.0.0: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
@ -500,6 +500,19 @@ rule "MAN.0.0: Apply manual resize redaction"
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$imageToBeResized: Image(id == $id)
then
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($imageToBeResized);
update($imageToBeResized.getParent());
end
// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
@ -540,12 +553,25 @@ rule "MAN.2.0: Apply force redaction"
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.2.1: Apply force redaction to images"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToForce: Image(id == $id)
then
$imageToForce.getManualOverwrite().addChange($force);
update($imageToForce);
update($imageToForce.getParent());
end
// Rule unit: MAN.3
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
@ -555,6 +581,39 @@ rule "MAN.3.0: Apply entity recategorization"
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply image recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
update($imageToBeRecategorized);
update($imageToBeRecategorized.getParent());
retract($recategorization);
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
end
rule "MAN.4.1: Apply legal basis change"
salience 128
when
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
end
//------------------------------------ Entity merging rules ------------------------------------

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl
View File

@ -1073,7 +1073,6 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
$dossierRedaction: TextEntity(type == "dossier_redaction")
then
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
update($dossierRedaction);
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
end
@ -1162,7 +1161,8 @@ rule "AI.2.0: add all NER Entities of any type except CBI_author"
rule "MAN.0.0: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
@ -1174,7 +1174,8 @@ rule "MAN.0.0: Apply manual resize redaction"
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$imageToBeResized: Image(id == $id)
then
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
@ -1240,7 +1241,8 @@ rule "MAN.2.1: Apply force redaction to images"
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
@ -1253,7 +1255,8 @@ rule "MAN.3.0: Apply entity recategorization"
rule "MAN.3.1: Apply image recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);

156
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1261,12 +1261,27 @@ rule "DOC.45.0: Doses (mg/kg bodyweight)"
rule "MAN.0.0: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId)
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$imageToBeResized: Image(id == $id)
then
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($imageToBeResized);
update($imageToBeResized.getParent());
end
@ -1274,50 +1289,99 @@ rule "MAN.0.0: Apply manual resize redaction"
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.removeFromGraph();
retract($entityToBeRemoved);
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
update($entityToBeRemoved);
retract($idRemoval);
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
$imageEntityToBeRemoved: Image($id == id)
then
$imageEntityToBeRemoved.setIgnored(true);
$imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
update($imageEntityToBeRemoved);
retract($idRemoval);
update($imageEntityToBeRemoved.getParent());
end
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.setRemoved(false);
$entityToForce.setIgnored(false);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
$entityToForce.getManualOverwrite().addChange($force);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
retract($force);
end
rule "MAN.2.1: Apply force redaction to images"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToForce: Image(id == $id)
then
$imageToForce.getManualOverwrite().addChange($force);
update($imageToForce);
update($imageToForce.getParent());
end
// Rule unit: MAN.3
rule "MAN.3.0: Apply image recategorization"
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
$image: Image($id == id)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
then
$image.setImageType(ImageType.fromString($imageType));
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply image recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
update($imageToBeRecategorized);
update($imageToBeRecategorized.getParent());
retract($recategorization);
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
end
rule "MAN.4.1: Apply legal basis change"
salience 128
when
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
end
@ -1328,14 +1392,64 @@ rule "MAN.3.0: Apply image recategorization"
rule "X.0.0: remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, getActive())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, getActive())
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
// Rule unit: X.2
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
// Rule unit: X.3
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
end
// Rule unit: X.4
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
retract($recommendation);
end
// Rule unit: X.5
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity(entityType == EntityType.ENTITY, active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation);
end
// Rule unit: X.7
rule "X.7.0: remove all images"
salience 512

15
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -65,7 +65,8 @@ query "getFileAttributes"
rule "MAN.0.0: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
@ -77,7 +78,8 @@ rule "MAN.0.0: Apply manual resize redaction"
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$imageToBeResized: Image(id == $id)
then
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
@ -143,11 +145,12 @@ rule "MAN.2.1: Apply force redaction to images"
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
@ -156,7 +159,8 @@ rule "MAN.3.0: Apply entity recategorization"
rule "MAN.3.1: Apply image recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
@ -184,7 +188,6 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -798,7 +798,6 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
$dossierRedaction: TextEntity(type == "dossier_redaction")
then
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
update($dossierRedaction);
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
end
@ -875,7 +874,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
rule "MAN.0.0: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
@ -887,7 +887,8 @@ rule "MAN.0.0: Apply manual resize redaction"
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$imageToBeResized: Image(id == $id)
then
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
@ -953,7 +954,8 @@ rule "MAN.2.1: Apply force redaction to images"
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
@ -966,7 +968,8 @@ rule "MAN.3.0: Apply entity recategorization"
rule "MAN.3.1: Apply image recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);

132
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -33,6 +33,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
@ -89,6 +90,136 @@ rule "Always redact PII"
end
//------------------------------------ Manual redaction rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$imageToBeResized: Image(id == $id)
then
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($imageToBeResized);
update($imageToBeResized.getParent());
end
// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
salience 128
when
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
update($entityToBeRemoved);
retract($idRemoval);
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
salience 128
when
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
$imageEntityToBeRemoved: Image($id == id)
then
$imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
update($imageEntityToBeRemoved);
retract($idRemoval);
update($imageEntityToBeRemoved.getParent());
end
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.2.1: Apply force redaction to images"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToForce: Image(id == $id)
then
$imageToForce.getManualOverwrite().addChange($force);
update($imageToForce);
update($imageToForce.getParent());
end
// Rule unit: MAN.3
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply image recategorization"
salience 128
when
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualImageRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
update($imageToBeRecategorized);
update($imageToBeRecategorized.getParent());
retract($recategorization);
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
end
rule "MAN.4.1: Apply legal basis change"
salience 128
when
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
end
//------------------------------------ Entity merging rules ------------------------------------
@ -211,4 +342,3 @@ rule "LDS.0.0: run local dictionary search"
entity.addMatchedRules(matchedRules);
});
end