diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 4bff8a03..6c63a1ac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -122,9 +122,7 @@ public class AnalyzeService { dictionary.getVersion(), kieWrapperEntityRules.rulesVersion()); - return finalizeAnalysis(analyzeRequest, - startTime, - kieWrapperComponentRules, new EntityLogChanges(entityLog, false), + return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), redactionLog, document.getNumberOfPages(), dictionary.getVersion(), @@ -160,10 +158,14 @@ public class AnalyzeService { log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (sectionsToReAnalyse.isEmpty()) { + return finalizeAnalysis(analyzeRequest, startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), - new EntityLogChanges(previousEntityLog, false), + entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog, + dictionaryIncrement.getDictionaryVersion(), + analyzeRequest.getDossierTemplateId(), + false), previousRedactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -195,10 +197,7 @@ public class AnalyzeService { RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, - document, - notFoundManualRedactionEntries, - previousEntityLog, - sectionsToReanalyseIds); + document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds, dictionary.getVersion()); return finalizeAnalysis(analyzeRequest, startTime, @@ -237,12 +236,7 @@ public class AnalyzeService { } - - - - private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, - long startTime, - KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, + private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, RedactionLog redactionLog, int numberOfPages, DictionaryVersion dictionaryVersion, @@ -436,5 +430,4 @@ public class AnalyzeService { } } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 9cd54d27..1081b4f2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -86,8 +86,7 @@ public class EntityLogCreatorService { public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, Document document, List notFoundManualRedactionEntries, - EntityLog previousEntityLog, - Set sectionsToReanalyseIds) { + EntityLog previousEntityLog, Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); List previousEntries = previousEntityLog.getEntityLogEntry() @@ -105,9 +104,22 @@ public class EntityLogCreatorService { previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries); previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); + excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages()); - return new EntityLogChanges(previousEntityLog, hasChanges); + return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges); + } + + + public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) { + + List legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId); + entityLog.setLegalBasisVersion(legalBasisClient.getVersion(dossierTemplateId)); + entityLog.setLegalBasis(toEntityLogLegalBasis(legalBasis)); + entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); + entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); + + return new EntityLogChanges(entityLog, hasChanges); } @@ -129,8 +141,7 @@ public class EntityLogCreatorService { List entries = new ArrayList<>(); document.getEntities() - .stream() - .filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active) + .stream().filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active) .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); document.streamAllImages().filter(IEntity::active).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); notFoundManualRedactionEntries.stream().filter(IEntity::active).forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); @@ -184,10 +195,14 @@ public class EntityLogCreatorService { .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .type(entity.getType()) .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) - .sectionNumber(sectionNumber).matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()).dictionaryEntry(entity.isDictionaryEntry()) + .sectionNumber(sectionNumber) + .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(entity.isDictionaryEntry()) .textAfter(entity.getTextAfter()) .textBefore(entity.getTextBefore()) - .startOffset(entity.getTextRange().start()).endOffset(entity.getTextRange().end()).dossierDictionaryEntry(entity.isDossierDictionaryEntry()) + .startOffset(entity.getTextRange().start()) + .endOffset(entity.getTextRange().end()) + .dossierDictionaryEntry(entity.isDossierDictionaryEntry()) .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) .reference(referenceIds) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) @@ -204,7 +219,9 @@ public class EntityLogCreatorService { return EntityLogEntry.builder() .id(manualEntity.getId()) .color(getColor(type, dossierTemplateId, manualEntity.applied())) - .reason(manualEntity.buildReasonWithManualChangeDescriptions()).legalBasis(manualEntity.legalBasis()).value(manualEntity.value()) + .reason(manualEntity.buildReasonWithManualChangeDescriptions()) + .legalBasis(manualEntity.legalBasis()) + .value(manualEntity.value()) .type(type) .state(buildEntryState(manualEntity)) .entryType(buildEntryType(manualEntity, isHint)) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java index c3b017bc..4da99ea9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java @@ -229,7 +229,7 @@ class DroolsSyntaxValidationServiceTest { continue; } RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index ea8db469..a92223b5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -639,7 +639,7 @@ rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active()) + $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -651,7 +651,7 @@ rule "X.1.0: merge intersecting Entities of same type" salience 64 when $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active()) + $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -667,7 +667,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -680,7 +680,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM salience 64 when $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -692,7 +692,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit salience 256 when $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); @@ -705,7 +705,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when $entity: TextEntity(entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); @@ -717,7 +717,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT salience 32 when $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl index edf7fb72..d8b1054c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl @@ -1,8 +1,8 @@ package drools import static java.lang.String.format; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.anyMatch; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.exactMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; import java.util.List; import java.util.LinkedList; @@ -12,45 +12,51 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; -import com.iqser.red.service.redaction.v1.server.document.graph.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.*; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; -import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity; global Document document global EntityCreationService entityCreationService -global ManualRedactionApplicationService manualRedactionApplicationService +global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary //------------------------------------ queries ------------------------------------ @@ -80,96 +86,7 @@ rule "H.0.0: Ignore Table of Contents" retract($node); end - -// Rule unit: MAN.0 -/* -rule "H.0.0: Show headlines" - when - $headline: Headline() - then - entityCreationService.bySemanticNode($headline, "headline", EntityType.ENTITY); - end -*/ - -/* -rule "NER.0.0: Show NER Countries" - when - $nerEntities: NerEntities(hasEntitiesOfType("COUNTRY")) - $section: Section() - then - $nerEntities - .streamEntitiesOfType("COUNTRY") - .map( - nerEntity -> entityCreationService - .byNerEntity( - nerEntity, - "ner_country", - EntityType.ENTITY, - $section - ) - ) - .forEach( - entity -> { - entity.skip("NER.0.0", "Country found by NER"); - insert(entity); - } - ); - end - -rule "NER.0.1: Show NER Cities" - when - $nerEntities: NerEntities(hasEntitiesOfType("CITY")) - $section: Section() - then - $nerEntities - .streamEntitiesOfType("CITY") - .map( - nerEntity -> entityCreationService - .byNerEntity( - nerEntity, - "ner_city", - EntityType.ENTITY, - $section - ) - ) - .forEach( - entity -> { - entity.skip("NER.0.1", "City found by NER"); - insert(entity); - } - ); - end - */ - -/* -rule "NER.0.0: Show NER Entities" - when - $nerEntities: NerEntities( - hasEntitiesOfType("STREET"), - (hasEntitiesOfType("CARDINAL") || hasEntitiesOfType("POSTAL") || hasEntitiesOfType("CITY") || hasEntitiesOfType("COUNTRY")) - ) - then - NerEntitiesAdapter.combineNerEntities( - $nerEntities, - Set.of("STREET"), - Set.of("STREET","CARDINAL","POSTAL","CITY","COUNTRY"), - 20, - 3, - false - ) - .forEach( - nerBoundary -> entityCreationService.byBoundary( - nerBoundary, - "ner", - EntityType.ENTITY, - document - ) - .ifPresent( - entity -> entity.skip("NER.0.0","NER address.") - ) - ); - end - */ +//---------------------- CUSTOMER RULES ----------------------------------------------------- rule "DOC.1.0: Adama number" when @@ -254,8 +171,6 @@ rule "DOC.2.1: Study number in header" ); end -// this rule breaks documents -/* rule "DOC.2.2: Project Number with Section Title" when $title: String() from List.of( @@ -276,7 +191,6 @@ rule "DOC.2.2: Project Number with Section Title" entity -> entity.apply("DOC.2.2","Study number by title found.") ); end - */ rule "DOC.3.0: Batch Material Number" @@ -322,51 +236,250 @@ rule "DOC.3.0: Batch Material Number" insert(FileAttribute.builder().label("Batch No").value(entity.getValue()).build()); } ); - // how can I unset the fileAttribute if there is no match any longer? end -rule "DOC.4.0: Study title by document structure" +rule "DOC.4.0: study title as headline" when - $headline: Headline(containsStringIgnoreCase("FINAL REPORT")) - $section: Section(getHeadline() == $headline && onPage(1)) - then - $section - .streamAllSubNodesOfType(NodeType.PARAGRAPH) - .findFirst() - .ifPresent( - paragraph -> entityCreationService - .bySemanticNode( - $headline, - "title", - EntityType.ENTITY - ) - .ifPresent( - entity -> entity.apply("DOC.4.0","Report title found.") - ) - ); - end - -rule "DOC.4.1: study title as headline" - when - not Table(onPage(1)) $hlKeyword: String() from List.of( "Analyitical Method", "Residues", - "Acute" + "Acute", + "Process", + "Application", + "Review", + "Examination", + "Toxicity", + "Method Validation", + "Explosion", + "Sensitisation", + "Determination", + "Test" ) + $hlNotKeyword: String() from List.of( + "Laboratoire", + "Laboratorien" + ) + $species: TextEntity(type=="species") $headline: Headline( onPage(1) - && containsStringIgnoreCase($hlKeyword) + && ( + containsStringIgnoreCase($hlKeyword) + || entities contains $species + ) + && !containsStringIgnoreCase($hlNotKeyword) ) then - entityCreationService.bySemanticNode($headline, "title", EntityType.ENTITY).ifPresent(entity -> - entity.apply("DOC.4.1", "Title found", "n-a") - ); + entityCreationService.bySemanticNode($headline, "title", EntityType.ENTITY).ifPresent(entity -> { + entity.apply("DOC.4.0", "Title found", "n-a"); + insert(FileAttribute.builder().label("Title").value(entity.getValue()).build()); + }); end +rule "DOC.4.1: study title on cover page between sections" + when + not TextEntity(type=="title", applied()) + $page: Page( + getNumber() == 1, + getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Title".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Title".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Report 92 50 12 136".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Report".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Study Report".toLowerCase()), + getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Guideline".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Identification".toLowerCase()) + // too many false positives due to term in header and cover page || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Report".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Data Requirement".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Submitted".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Test Guideline".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Director".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Author".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Including:".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Laboratory Investigations".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Test Article".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "HLS".toLowerCase()) + || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Official Journal".toLowerCase()) + ) + then + List startStrings = List.of("Study Title", "Study Title:", "Title", "Final Report", "Final Study Report", "Report 92 50 12 136"); -rule "DOC.5.0: Author with headline or on pages 1 and 2" + + List stopStrings = List.of("Guideline", "Guidelines", "Study Identification", "Data Requirement", "Submitted", "Test Guideline", "Study Director", "Author", "Including:", "Laboratory Investigations", "Test Article", "HLS", "Official Journal"); + // too many false positives due to term in header and cover page stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Final Report", $page.getMainBodyTextBlock())); + + entityCreationService.shortestBetweenAnyString(startStrings, stopStrings, "title", EntityType.ENTITY, document).forEach(entity -> { + entity.apply("DOC.4.1", "Study title found", "n-a"); + insert(FileAttribute.builder().label("Title").value(entity.getValue()).build()); + }); + end + +rule "DOC.4.2: Study title by paragraph including species on cover page" + when + not TextEntity(type=="title", applied()) + $species: TextEntity(type=="species") + $paragraph: Paragraph( + onPage(1) + && entities contains $species + ) + then + entityCreationService.bySemanticNode( + $paragraph, + "title", + EntityType.ENTITY + ) + .ifPresent(entity -> { + entity.apply("DOC.4.2","Title with species found on cover page"); + insert(FileAttribute.builder().label("Title").value(entity.getValue()).build()); + }); + end + +rule "DOC.5.0: Study Director on cover page in a paragraph on line" + when + not TextEntity(type=="author", applied()) + $sectionTitle: String() from List.of( + "Study Director/Analyst:", + "Study Director:", + "Study Director :", + "Author:", + "Author " + ) + $paragraph: Paragraph( + onPage(1) + && containsStringIgnoreCase($sectionTitle) + ) + then + entityCreationService + .lineAfterStringIgnoreCase( + $sectionTitle, + "author", + EntityType.ENTITY, + $paragraph + ) + .forEach(entity -> { + entity.apply("DOC.5.0","Study Director on cover page found"); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + }); + end + +rule "DOC.5.1: Author/Study Director listing on cover page" + when + not TextEntity(type=="author", applied()) + $page: Page( + getNumber() == 1 + && getMainBodyTextBlock().getSearchText().toLowerCase() ( + contains "Author(s)".toLowerCase() + || contains "Authors".toLowerCase() + || contains "Author".toLowerCase() + || contains "Study Director".toLowerCase() + ) + && getMainBodyTextBlock().getSearchText().toLowerCase() ( + contains "Test Facility".toLowerCase() + || contains "Testing Facility".toLowerCase() + || contains "Study Initiation".toLowerCase() + || contains "Study Initiated".toLowerCase() + || contains "Study completed on".toLowerCase() + || contains "Study completion".toLowerCase() + || contains "Study amended".toLowerCase() + || contains "Report Issue Date".toLowerCase() + || contains "Document Date".toLowerCase() + || contains "Date".toLowerCase() + || contains "Defitrace".toLowerCase() // this should be solved with Organisation Entity as a TextRange + ) + ) + then + List startBoundaries = new LinkedList<>(); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author(s)", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Authors", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Director", $page.getMainBodyTextBlock())); + + List stopBoundaries = new LinkedList<>(); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Test Facility", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Testing Facility", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Initiation", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Initiated", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study completed on", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study completion", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study amended", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Report Issue Date", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Document Date", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Date", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Defitrace", $page.getMainBodyTextBlock())); + + entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "author", EntityType.ENTITY, document).forEach(entity -> { + entity.apply("DOC.5.1", "Author list found", "n-a"); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + }); + end + +rule "DOC.5.2: Study Director on cover page in a paragraph" + when + not TextEntity(type=="author", applied()) + $sectionTitle: String() from List.of( + "Study Director" + ) + $node: Paragraph( + onPage(1) + && containsStringIgnoreCase($sectionTitle) + ) + then + entityCreationService + .byRegexWithLineBreaksIgnoreCase( + "\\n([\\w\\(\\) .]{5,30})\\n", + "author", + EntityType.ENTITY, + 1, + $node + ) + .filter( + entity -> !entity.getValue().toLowerCase().contains($sectionTitle.toLowerCase()) + ) + .forEach(entity -> { + entity.apply("DOC.5.2","Study Director on cover page found"); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + }); + end + +rule "DOC.5.3: Study Director on cover page in a section" + when + not TextEntity(type=="author", applied()) + $page: Page(getNumber() == 1) + $sectionTitle: String() from List.of( + "Study Director:", + "Study Director", + "Study Director/Author" + ) + $node: Section( + onPage(1) + && containsStringIgnoreCase($sectionTitle) + ) + then + entityCreationService + .byRegexWithLineBreaksIgnoreCase( + $sectionTitle+"\\n([\\w\\(\\) .]{5,30})\\n", + "author", + EntityType.ENTITY, + 1, + $node + ) + .filter( + entity -> entity.getPages().contains($page) + ) + .forEach(entity -> { + entity.apply("DOC.5.3","Study Director on cover page found"); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + }); + + entityCreationService + .lineAfterStringIgnoreCase($sectionTitle,"author",EntityType.ENTITY,$node) + .forEach(entity -> { + entity.apply("DOC.5.2","Study Director on cover page found"); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + }); + end + +rule "DOC.5.4: Author in document section with relevant headline" when + not TextEntity(type == "author", applied()) $hlKeyword: String() from List.of( "sponsor", "personnel", @@ -381,15 +494,12 @@ rule "DOC.5.0: Author with headline or on pages 1 and 2" "Study Director :", "stray Birector:", "Author:", - "Author", + "Author ", "Author(s)" ) $section: Section( containsStringIgnoreCase($keyword) - && ( - getHeadline().containsStringIgnoreCase($hlKeyword) - || (onPage(1) || onPage(2)) - ) + && getHeadline().containsStringIgnoreCase($hlKeyword) ) then entityCreationService @@ -402,381 +512,99 @@ rule "DOC.5.0: Author with headline or on pages 1 and 2" .findFirst() .ifPresent( entity -> { - entity.apply("DOC.5.0","Author found."); + entity.apply("DOC.5.4","Author found."); insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); } ); end -// separate rule as I cannot save a righthand side with two entityCreationService-calls -/* -rule "DOC.5.1: Author" - when - $keyword: String() from List.of( - "Author(s)", - "Author", - "Authors" - ) - $stopKeyword: String() from List.of( - "Test Facility", - "Study Initiation", - "Study completed on" - ) - $author: RedactionEntity(type=="author") - $section: Section( - containsStringIgnoreCase($keyword) - && containsStringIgnoreCase($stopKeyword) - && onPage(1) - ) - then - // betweenStringsIgnoreCase captures the longest possible string (should be the shortest?) - entityCreationService - .betweenStrings( - $keyword, - $stopKeyword, - "author", - EntityType.ENTITY, - $section - ) - .forEach( - entity -> { - entity.apply("DOC.5.1","Author found."); - insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); - } - ); - end -*/ - -rule "DOC.5.2: Authors list on cover page paragraph" +rule "DOC.5.5: Study Director from signature on compliance statement" when + not TextEntity(type=="author", applied()) $sectionTitle: String() from List.of( - "Study Director" + "Compliance Statement", + "Study Compliance", + "Statement of Compliance", + "Statement of Study Compliance", + "Contributing Scientist", + "Compliance with Good Laboratory Practice", + "Signatures and Approval" ) - $author: RedactionEntity(type=="author", isApplied()) - $node: Paragraph( - onPage(1) - && entities not contains $author - && containsStringIgnoreCase($sectionTitle) + $role: String() from List.of("Study Director") + $node: Section( + containsStringIgnoreCase($sectionTitle) + && containsStringIgnoreCase($role) + && getFirstPage().getNumber() > 4 ) then entityCreationService .byRegexWithLineBreaksIgnoreCase( - "([\\w\\(\\) .]{5,30})\\n", + "([\\w\\(\\) .,]{5,35})[\\w\\/ ]{0,30}\\n"+$role, "author", EntityType.ENTITY, 1, $node ) - .filter( - entity -> !entity.getValue().toLowerCase().contains($sectionTitle.toLowerCase()) - ) - .forEach( - entity -> entity.apply("DOC.5.2","Author list on cover page found") - ); + .forEach(entity -> { + entity.apply("DOC.5.5","Study Director on compliance page found"); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + }); end -rule "DOC.5.3: Authors list on cover page section" +rule "DOC.5.6: Study Director from section with relevant headlining responsibility" when - $author: RedactionEntity(type=="author", isApplied()) - $sectionTitle: String() from List.of( - "Study Director" - ) + not TextEntity(type=="author", applied()) + $sectionTitle: String() from List.of("Responsible personnel") + $role: String() from List.of("Study Director") $node: Section( - onPage(1) - && entities not contains $author - && containsStringIgnoreCase($sectionTitle) + getHeadline().containsStringIgnoreCase($sectionTitle) + && containsStringIgnoreCase($role) + ) + $paragraph: Paragraph( + containsStringIgnoreCase($role) ) then entityCreationService - .byRegexWithLineBreaksIgnoreCase($sectionTitle + "\\n([\\w\\(\\) .]{5,30})\\n","author",EntityType.ENTITY,1,$node) - .forEach( - entity -> entity.apply("DOC.5.3","Author list on cover page found") - ); + .lineAfterStringIgnoreCase( + $role, + "author", + EntityType.ENTITY, + $paragraph + ) + .forEach(entity -> { + entity.apply("DOC.5.6","Study Director on compliance page found"); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + }); + end + +rule "DOC.5.7: Study Director from section with study director in headline" + when + not TextEntity(type=="author", applied()) + $sectionTitle: String() from List.of("Study Director") + $section: Section( + getHeadline().containsStringIgnoreCase($sectionTitle) + ) + then + entityCreationService + .bySemanticNodeParagraphsOnly( + $section, + "author", + EntityType.ENTITY + ) + .forEach(entity -> { + entity.apply("DOC.5.7","Study Director found by headline"); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + }); end rule "DOC.5.9: Remove skipped authors" salience 9999 when - $author: RedactionEntity(type=="author", !isApplied()) + $author: TextEntity(type=="author", !applied()) then $author.removeFromGraph(); end -/* -rule "DOC.6.0: Finalization Date" - when - $headline: Headline( - containsStringIgnoreCase("study dates") - ) - $section: Section( - (getHeadline() == $headline) - && containsStringIgnoreCase("final report") - ) - then - entityCreationService - .lineAfterStrings( - List.of( - "final report" - ), - "finalization_date", - EntityType.ENTITY, - $section - ) - .forEach( - entity -> entity.apply("DOC.6.0", "Experimental end date found", "n-a") - ); - end - -rule "DOC.7.0: Test on vertebrates" - when - $headline: Headline( - containsStringIgnoreCase("animal") - ) - $species: RedactionEntity(type=="species") - $paragraph: Paragraph( - getHeadline() == $headline - && entities contains $species - && containsStringIgnoreCase("species") - ) - then - $species.apply("DOC.7.0","Species found in animal section"); - end - -rule "DOC.8.0: Testing Facility name using NER entity boundaries" - when - $nerEntities: NerEntities( - hasEntitiesOfType("STREET"), - (hasEntitiesOfType("CARDINAL") || hasEntitiesOfType("POSTAL") || hasEntitiesOfType("CITY") || hasEntitiesOfType("COUNTRY")) - ) - $testFacilityKeyword: String() from List.of( - "Test facility", - "Test Facility:", - "Testing Facility:" - ) - $paragraph: Paragraph( - containsStringIgnoreCase("facility") - ) - then - entityCreationService.byString( - $testFacilityKeyword, - "hint", - EntityType.ENTITY, - $paragraph - ) - .forEach( - ent -> ent.apply("DOC.8.0","Facility hint") - ); - */ - /* - // Needs BugFixing - Documents with NerEntities in $paragraph - // switch to state "Re-processing required" - entityCreationService.betweenBoundaries( - RedactionSearchUtility.findBoundariesByStringIgnoreCase( - $testFacilityKeyword, - $paragraph.getTextBlock() - ), - NerEntitiesAdapter.combineNerEntities( - $nerEntities, - Set.of("STREET"), - Set.of("STREET","CARDINAL","POSTAL","CITY","COUNTRY"), - 20, - 3, - false - ).toList(), - "testing_facility_name", - EntityType.ENTITY, - $paragraph - ) - .forEach( - entity -> entity.apply("DOC.8.0","Test Facility name found") - ); - */ - /* - end - -rule "DOC.8.2: Performing Laboratory (Country & Name) from dict" - when - $facilityCountry: RedactionEntity(type=="testing_facility_country") - $facilityName: RedactionEntity(type=="testing_facility_name") - $section: Section( - ( - entities contains $facilityCountry - || entities contains $facilityName - ) - && ( - containsString("Testing Facility:") - || ( - containsString("Testing") - && containsString("Facility:") - ) - ) - ) - then - $section.getEntitiesOfType("testing_facility_country").forEach(entity -> { - entity.apply("DOC.8.2", "Testing facility country dictionary entry found."); - }); - $section.getEntitiesOfType("testing_facility_name").forEach(entity -> { - entity.apply("DOC.8.2", "Testing facility dictionary entry found."); - }); - end - - -rule "DOC.8.3: Performing Laboratory (Country) from dict" - when - $section: Section( - (hasEntitiesOfType("testing_facility_country") || hasEntitiesOfType("testing_facility_name")) - && !(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:"))) - ) - then - $section.getEntitiesOfType(List.of("testing_facility_country", "testing_facility_name")).forEach(entity -> { - entity.removeFromGraph(); - retract(entity); - }); - end - -rule "DOC.8.4: Test Facility Country with Ner Entities" - when - $countries: RedactionEntity(type=="ner_country") - $headline: Headline( - containsStringIgnoreCase("test facility") - ) - $section: Section( - getHeadline() == $headline - ) - $paragraph: Paragraph( - containsStringIgnoreCase("test facility") - && entities contains $countries - ) from $section.streamAllSubNodesOfType(NodeType.PARAGRAPH).toList() - then - $paragraph - .getEntitiesOfType("ner_country") - .forEach( - entity -> entityCreationService - .byBoundary( - entity.getBoundary(), - "testing_facility_country", - EntityType.ENTITY, - $paragraph - ) - .ifPresent( - country -> country.apply("DOC.8.4","Testing Facility Country found by NER") - ) - ); - end - -rule "DOC.9.0: Sponsor Name" - when - $headline: Headline( - containsStringIgnoreCase("sponsor") - ) - $section: Section( - getHeadline() == $headline - ) - $paragraph: Paragraph( - containsStringIgnoreCase("sponsor") - ) from $section.streamAllSubNodesOfType(NodeType.PARAGRAPH).toList() - then - entityCreationService - .semanticNodeAfterString( - "Sponsor", - "sponsor_name", - EntityType.ENTITY, - $paragraph - ) - .ifPresent( - entity -> entity.apply("DOC.9.0", "Sponsor found") - ); - end - - -rule "DOC.10.0: Active Ingredient Name" - when - $paragraph: Paragraph( - getHeadline().containsStringIgnoreCase("description") - && containsStringIgnoreCase("Content") - ) - then - entityCreationService - .byRegexIgnoreCase( - "\\b(\\w{5,20})\\b[\\s\\d.]{3,6}%", - "active_ingredient_name", - EntityType.ENTITY, - 1, - $paragraph - ) - .forEach( - entity -> entity.apply("DOC.10.0","Content names found.") - ); - end - -rule "DOC.11.0: Product Names" - when - $paragraph: Paragraph( - getHeadline().containsStringIgnoreCase("description") - && containsStringIgnoreCase("Content") - ) - then - entityCreationService - .byRegexIgnoreCase( - "\\b(\\w{5,20})\\b[\\s\\d.]{3,6}%", - "product_name", - EntityType.ENTITY, - 1, - $paragraph - ) - .forEach( - entity -> entity.apply("DOC.11.0","Product names found.") - ); - end - -rule "DOC.12.0: Section" - when - then - end - -rule "DOC.13.0: Guidelines" - when - $section: Section( - ( - containsString("according to") - || containsString("method") - ) - && ( - containsString("OECD") - || containsString("EPA") - || containsString("OPPTS") - || containsString("EC") - ) - ) - then - entityCreationService.byRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", "guideline", EntityType.ENTITY, $section).forEach(entity -> - entity.apply("DOC.13.0", "OECD Guideline found", "n-a") - ); - entityCreationService.byRegex("OECD[\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\d{3,4}(?>.{1,100}\\d{4}\\))?|\\[.{1,20}.Skin.{1,20}\\]|[\\d\\s,\\(\\)]{7,10}|[\\w\\.\\s]{1,15}[\\d]{3}\\s\\(\\d{4}\\)|.{0,20}[N|n]umber\\s\\d{3}.{0,1}|Test Guideline \\d{3})", "guideline", EntityType.ENTITY, $section).forEach(entity -> - entity.apply("DOC.13.0", "OECD Guideline found", "n-a") - ); - entityCreationService.byRegex("OECD [G|g]uideline 4\\d{2}", "guideline", EntityType.ENTITY, $section).forEach(entity -> - entity.apply("DOC.13.0", "OECD Guideline found", "n-a") - ); - - entityCreationService.byRegex("EC (Directive )?(No\\.? )?\\d{3,4}\\/\\d{3,4}((,? B(\\.| )\\d{1,2}\\.?)? \\(\\d{4}\\))?", "guideline", EntityType.ENTITY, $section).forEach(entity -> - entity.apply("DOC.13.0", "EC Guideline found", "n-a") - ); - entityCreationService.byRegex("EC method B(\\.| )\\d{1,2} ?\\w{0,5} \\([\\d\\/EC]{4,10}\\)?", "guideline", EntityType.ENTITY, $section).forEach(entity -> - entity.apply("DOC.13.0", "EC Guideline found", "n-a") - ); - entityCreationService.byRegex("Commission Regulation \\(EC\\) No \\d{3}\\/\\d{4}", "guideline", EntityType.ENTITY, $section).forEach(entity -> - entity.apply("DOC.13.0", "EC Guideline found", "n-a") - ); - - entityCreationService.byRegex("OPPTS (Guideline Number )?\\d{3}\\.\\d{4}( \\(\\d{4}\\))?", "guideline", EntityType.ENTITY, $section).forEach(entity -> - entity.apply("DOC.13.0", "EPA Guideline found", "n-a") - ); - entityCreationService.byRegex("EPA (OPPTS )?\\d{3}[. ]\\d{4}( \\(\\d{4}\\))?", "guideline", EntityType.ENTITY, $section).forEach(entity -> - entity.apply("DOC.13.0", "EPA Guideline found", "n-a") - ); - end -*/ rule "DOC.14.0: GLP Study" when $hlKeywords: String() from List.of( @@ -847,12 +675,27 @@ rule "DOC.14.1: GLP Study" rule "MAN.0.0: Apply manual resize redaction" salience 128 when - $resizeRedaction: ManualResizeRedaction($id: annotationId) - $entityToBeResized: RedactionEntity(matchesAnnotationId($id)) + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); + $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $imageToBeResized: Image(id == $id) + then + manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); end @@ -860,22 +703,25 @@ rule "MAN.0.0: Apply manual resize redaction" rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) - not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) - $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.removeFromGraph(); - retract($entityToBeRemoved); + $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) - not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) $imageEntityToBeRemoved: Image($id == id) then - $imageEntityToBeRemoved.setIgnored(true); + $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); end @@ -883,30 +729,89 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to rule "MAN.2.0: Apply force redaction" salience 128 when - $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) - $entityToForce: RedactionEntity(matchesAnnotationId($id)) + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); - $entityToForce.setRemoved(false); - $entityToForce.setIgnored(false); - $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + $entityToForce.getManualOverwrite().addChange($force); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end - -// Rule unit: MAN.3 -rule "MAN.3.0: Apply image recategorization" +rule "MAN.2.1: Apply force redaction to images" salience 128 when - ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) - $image: Image($id == id) + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToForce: Image(id == $id) then - $image.setImageType(ImageType.fromString($imageType)); + $imageToForce.getManualOverwrite().addChange($force); + update($imageToForce); + update($imageToForce.getParent()); + retract($force); end +// Rule unit: MAN.3 +rule "MAN.3.0: Apply entity recategorization" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + then + $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); + manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + retract($recategorization); + // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. + retract($entityToBeRecategorized); + end + + +rule "MAN.3.1: Apply entity recategorization of same type" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + then + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + retract($recategorization); + end + + +rule "MAN.3.2: Apply image recategorization" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $imageToBeRecategorized: Image($id == id) + then + manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); + update($imageToBeRecategorized); + update($imageToBeRecategorized.getParent()); + retract($recategorization); + end + + +// Rule unit: MAN.4 +rule "MAN.4.0: Apply legal basis change" + salience 128 + when + $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToBeRecategorized: Image($id == id) + then + $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + end + +rule "MAN.4.1: Apply legal basis change" + salience 128 + when + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeChanged: TextEntity(matchesAnnotationId($id)) + then + $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + end + //------------------------------------ Entity merging rules ------------------------------------ @@ -914,14 +819,64 @@ rule "MAN.3.0: Apply image recategorization" rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when - $larger: RedactionEntity($type: type, $entityType: entityType, isActive()) - $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) + $larger: TextEntity($type: type, $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end +// Rule unit: X.2 +rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 64 + when + $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); + retract($entity) + end + + +// Rule unit: X.3 +rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 64 + when + $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); + retract($recommendation); + end + + +// Rule unit: X.4 +rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" + salience 256 + when + $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) + $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $entity.addEngines($recommendation.getEngines()); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); + retract($recommendation); + end + + +// Rule unit: X.5 +rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 256 + when + $entity: TextEntity(entityType == EntityType.ENTITY, active()) + $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); + retract($recommendation); + end + + // Rule unit: X.7 rule "X.7.0: remove all images" salience 512 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index 7b3fdeb2..7947e2c1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -1312,7 +1312,7 @@ rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active()) + $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -1324,7 +1324,7 @@ rule "X.1.0: merge intersecting Entities of same type" salience 64 when $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active()) + $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -1340,7 +1340,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1353,7 +1353,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM salience 64 when $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -1365,7 +1365,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit salience 256 when $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); @@ -1378,7 +1378,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when $entity: TextEntity(entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); @@ -1390,7 +1390,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT salience 32 when $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index ca78861e..326ec246 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -247,7 +247,7 @@ rule "DOC.1.3: Guidelines" ) then $section.getEntitiesOfType(List.of("oecd_guideline", "ec_guideline", "epa_guideline")).forEach(entity -> { - entity.removeFromGraph(); + entity.remove("DOC.1.3", "removed by Guidelines rules"); retract(entity); }); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl index b6e6b2df..6f4c71a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl @@ -91,9 +91,9 @@ rule "ReportNumber.0.0: Report number" rule "GLPStudy.0.0: GLP Study" when - $glpStudy: Entity(type == "glp_study") + $glpStudyList: List(!isEmpty) from collect(Entity(type == "glp_study")) then - componentCreationService.create("GLPStudy.0.0", "GLP_study", "Yes", "Yes if present, No if not", $glpStudy); + componentCreationService.create("GLPStudy.0.0", "GLP_study", "Yes", "Yes if present, No if not", $glpStudyList); end rule "GLPStudy.1.0: GLP Study" @@ -103,15 +103,6 @@ rule "GLPStudy.1.0: GLP Study" componentCreationService.create("GLPStudy.1.0", "GLP_study", "No", "Yes if present, No if not"); end -rule "GLPStudy.2.0: GLP Study" - when - $first: Component(category == "GLP_study") - $second: Component(category == "GLP_study", value == $first.value, this != $first) - then - $first.addReferences($second.getReferences()); - retract($second); - end - rule "TestGuideline.0.0: no test guideline year found, only oecd number" when @@ -173,12 +164,12 @@ rule "TestGuideline.1.1: match test guidelines with mappings" end -rule "DefaultComponents.5.0: Test Guideline 2" +rule "TestGuideline.2.0: Test Guideline 2" when $epaGuideLines: List() from collect (Entity(type == "epa_guideline")) $ecGuideLines: List() from collect (Entity(type == "ec_guideline")) then - componentCreationService.joining("DefaultComponents.5.0", + componentCreationService.joining("TestGuideline.2.0", "Test_Guideline_2", Stream.of( $epaGuideLines.stream(), diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 94acedec..ae144708 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -123,7 +123,6 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" - no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) @@ -132,10 +131,10 @@ rule "MAN.2.0: Apply force redaction" $entityToForce.getManualOverwrite().addChange($force); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); end rule "MAN.2.1: Apply force redaction to images" - no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) @@ -144,6 +143,7 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce.getManualOverwrite().addChange($force); update($imageToForce); update($imageToForce.getParent()); + retract($force); end @@ -151,9 +151,9 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); @@ -162,7 +162,20 @@ rule "MAN.3.0: Apply entity recategorization" retract($entityToBeRecategorized); end -rule "MAN.3.1: Apply image recategorization" + +rule "MAN.3.1: Apply entity recategorization of same type" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + then + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + retract($recategorization); + end + + +rule "MAN.3.2: Apply image recategorization" salience 128 when $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) @@ -175,6 +188,7 @@ rule "MAN.3.1: Apply image recategorization" retract($recategorization); end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 @@ -193,7 +207,6 @@ rule "MAN.4.1: Apply legal basis change" then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); end - //------------------------------------ Local dictionary search rules ------------------------------------ // Rule unit: LDS.0 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 7227d3b1..1f3bdff7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1023,7 +1023,7 @@ rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active()) + $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -1035,7 +1035,7 @@ rule "X.1.0: merge intersecting Entities of same type" salience 64 when $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active()) + $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -1051,7 +1051,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1064,7 +1064,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM salience 64 when $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -1076,7 +1076,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit salience 256 when $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); @@ -1089,7 +1089,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when $entity: TextEntity(entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); @@ -1101,7 +1101,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT salience 32 when $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");