From 6c3cf68fbf0274c2d07bceecf5849ede4cea2d29 Mon Sep 17 00:00:00 2001 From: Ali Oezyetimoglu Date: Fri, 10 Nov 2023 13:18:18 +0100 Subject: [PATCH] RED-7679: updated method rowValueCount to count rows with provided entity --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../document/ComponentCreationService.java | 5 +- .../v1/server/ComponentCreationTest.java | 118 ++++++ .../test/resources/drools/test_components.drl | 55 +++ .../src/test/resources/drools/test_rules.drl | 401 ++++++++++++++++++ 5 files changed, 577 insertions(+), 4 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/ComponentCreationTest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index cc87a214..5ecbd3cb 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -7,7 +7,7 @@ description = "redaction-service-api-v1" dependencies { implementation("org.springframework:spring-web:6.0.12") - implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.207.0") + implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.240.0") } publishing { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java index 8cf6e017..fcc8b834 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java @@ -307,15 +307,14 @@ public class ComponentCreationService { */ public void rowValueCount(String ruleIdentifier, String name, Collection entities) { - entities.stream().collect(Collectors.groupingBy(this::getFirstTableCell)).forEach((optionalTable, groupedEntities) -> { + entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> { if (optionalTable.isEmpty()) { return; } long count = groupedEntities.stream() - .filter(entity -> !(entity.getContainingNode() instanceof Paragraph) && entity.getContainingNode() instanceof TableCell) - .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow())) + .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1))) .size(); create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/ComponentCreationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/ComponentCreationTest.java new file mode 100644 index 00000000..0e46ab85 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/ComponentCreationTest.java @@ -0,0 +1,118 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; +import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"}) +@Import(ComponentCreationTest.RedactionIntegrationTestConfiguration.class) +public class ComponentCreationTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/test_rules.drl"); + private static final String COMPONENT_RULES = loadFromClassPath("drools/test_components.drl"); + + + @Test + public void testRowValueCount() throws IOException { + + AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); + + System.out.println("Start Full integration test"); + analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request); + System.out.println("Finished structure analysis"); + AnalyzeResult result = analyzeService.analyze(request); + System.out.println("Finished analysis"); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); + + String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf"; + + try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { + fileOutputStream.write(annotateResponse.getDocument()); + } + } + + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import(LayoutParsingServiceProcessorConfiguration.class) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + static class RedactionIntegrationTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("documine"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES)); + + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl new file mode 100644 index 00000000..72b65851 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl @@ -0,0 +1,55 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; + +global ComponentCreationService componentCreationService + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +query "getComponents" + $component: Component() + end + +//------------------------------------ Guideline mapping object ------------------------------------ + +declare GuidelineMapping + number: String + year: String + guideline: String + end + +//------------------------------------ Test Components rules ------------------------------------ + +rule "AdvTableExtr.2.3: Cells containing dose for survived males" + when + $tableValues: List() from collect (Entity(type == "test")) + then + componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl new file mode 100644 index 00000000..ccf3005a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -0,0 +1,401 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; + +global Document document +global EntityCreationService entityCreationService +global ManualChangesApplicationService manualChangesApplicationService +global Dictionary dictionary + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +//------------------------------------ Local dictionary search rules ------------------------------------ + +// Rule unit: LocalDictionarySearch.0 +rule "LDS.0.0: Run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) + .forEach(entity -> { + Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); + entity.addMatchedRules(matchedRules); + }); + end +// --------------------------------------- Your rules below this line -------------------------------------------------- + +rule "TAB.0.0: Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)" + when + $section: Section(containsString("Maximum occurrence")) + $table: Table() from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("water"), $row: row) from $table.streamTableCells().toList() + $test: TableCell($row == row) from $table.streamTableCells().toList() + then + System.out.println("AAAA: " + $test); + entityCreationService.bySemanticNode($test, "test", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Some test stuff")); + end + + +//------------------------------------ Manual redaction rules ------------------------------------ + +// Rule unit: MAN.0 +rule "MAN.0.0: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $entityToBeResized: TextEntity(matchesAnnotationId($id)) + then + manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($entityToBeResized); + $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $imageToBeResized: Image(id == $id) + then + manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); + end + + +// Rule unit: MAN.1 +rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" + salience 128 + when + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) + then + $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" + salience 128 + when + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) + $imageEntityToBeRemoved: Image($id == id) + then + $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); + end + + +// Rule unit: MAN.2 +rule "MAN.2.0: Apply force redaction" + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToForce: TextEntity(matchesAnnotationId($id)) + then + $entityToForce.getManualOverwrite().addChange($force); + update($entityToForce); + $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); + end + +rule "MAN.2.1: Apply force redaction to images" + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToForce: Image(id == $id) + then + $imageToForce.getManualOverwrite().addChange($force); + update($imageToForce); + update($imageToForce.getParent()); + retract($force); + end + + +// Rule unit: MAN.3 +rule "MAN.3.0: Apply entity recategorization" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + then + $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); + manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + retract($recategorization); + // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. + retract($entityToBeRecategorized); + end + +rule "MAN.3.1: Apply entity recategorization of same type" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + then + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + retract($recategorization); + end + +rule "MAN.3.2: Apply image recategorization" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $imageToBeRecategorized: Image($id == id) + then + manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); + update($imageToBeRecategorized); + update($imageToBeRecategorized.getParent()); + retract($recategorization); + end + + +// Rule unit: MAN.4 +rule "MAN.4.0: Apply legal basis change" + salience 128 + when + $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToBeRecategorized: Image($id == id) + then + $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + end + +rule "MAN.4.1: Apply legal basis change" + salience 128 + when + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeChanged: TextEntity(matchesAnnotationId($id)) + then + $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + end + + +//------------------------------------ Entity merging rules ------------------------------------ + +// Rule unit: X.0 +rule "X.0.0: Remove Entity contained by Entity of same type" + salience 65 + when + $larger: TextEntity($type: type, $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + then + $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); + retract($contained); + end + + +// Rule unit: X.1 +rule "X.1.0: Merge intersecting Entities of same type" + salience 64 + when + $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) + then + TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); + $first.remove("X.1.0", "merge intersecting Entities of same type"); + $second.remove("X.1.0", "merge intersecting Entities of same type"); + retract($first); + retract($second); + mergedEntity.getIntersectingNodes().forEach(node -> update(node)); + end + + +// Rule unit: X.2 +rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 64 + when + $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); + retract($entity) + end + + +// Rule unit: X.3 +rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 64 + when + $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); + retract($recommendation); + end + + +// Rule unit: X.4 +rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" + salience 256 + when + $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $entity.addEngines($recommendation.getEngines()); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); + retract($recommendation); + end + + +// Rule unit: X.5 +rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 256 + when + $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); + retract($recommendation); + end + + +// Rule unit: X.6 +rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" + salience 32 + when + $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + then + $lowerRank.getIntersectingNodes().forEach(node -> update(node)); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); + retract($lowerRank); + end + + +rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" + salience 32 + when + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + then + $higherRank.getIntersectingNodes().forEach(node -> update(node)); + $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); + retract($higherRank); + end + + +// Rule unit: X.7 +rule "X.7.0: remove all images" + salience 512 + when + $image: Image(imageType != ImageType.OCR, !hasManualChanges()) + then + $image.remove("X.7.0", "remove all images"); + retract($image); + end + + +//------------------------------------ File attributes rules ------------------------------------ + +// Rule unit: FA.1 +rule "FA.1.0: Remove duplicate FileAttributes" + salience 64 + when + $fileAttribute: FileAttribute($label: label, $value: value) + $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value) + then + retract($duplicate); + end +