RED-7679: updated method rowValueCount to count rows with provided entity
This commit is contained in:
Ali Oezyetimoglu
parent
191b73986b
commit
6c3cf68fbf
@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
|
||||
|
||||
dependencies {
|
||||
implementation("org.springframework:spring-web:6.0.12")
|
||||
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.207.0")
|
||||
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.240.0")
|
||||
}
|
||||
|
||||
publishing {
|
||||
|
||||
@ -307,15 +307,14 @@ public class ComponentCreationService {
|
||||
*/
|
||||
public void rowValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
|
||||
|
||||
entities.stream().collect(Collectors.groupingBy(this::getFirstTableCell)).forEach((optionalTable, groupedEntities) -> {
|
||||
entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
|
||||
|
||||
if (optionalTable.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
long count = groupedEntities.stream()
|
||||
.filter(entity -> !(entity.getContainingNode() instanceof Paragraph) && entity.getContainingNode() instanceof TableCell)
|
||||
.collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow()))
|
||||
.collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1)))
|
||||
.size();
|
||||
|
||||
create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities);
|
||||
|
||||
@ -0,0 +1,118 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.FilterType;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
|
||||
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
|
||||
@Import(ComponentCreationTest.RedactionIntegrationTestConfiguration.class)
|
||||
public class ComponentCreationTest extends AbstractRedactionIntegrationTest {
|
||||
|
||||
private static final String RULES = loadFromClassPath("drools/test_rules.drl");
|
||||
private static final String COMPONENT_RULES = loadFromClassPath("drools/test_components.drl");
|
||||
|
||||
|
||||
@Test
|
||||
public void testRowValueCount() throws IOException {
|
||||
|
||||
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
|
||||
|
||||
System.out.println("Start Full integration test");
|
||||
analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
|
||||
System.out.println("Finished structure analysis");
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
System.out.println("Finished analysis");
|
||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf";
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||
@Import(LayoutParsingServiceProcessorConfiguration.class)
|
||||
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
|
||||
static class RedactionIntegrationTestConfiguration {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public StorageService inmemoryStorage() {
|
||||
|
||||
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@BeforeEach
|
||||
public void stubClients() {
|
||||
|
||||
TenantContext.setTenantId("documine");
|
||||
|
||||
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
|
||||
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
|
||||
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis());
|
||||
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES));
|
||||
|
||||
loadTypeForTest();
|
||||
loadNerForTest();
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
|
||||
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.type(DOSSIER_REDACTIONS_INDICATOR)
|
||||
.dossierTemplateId(TEST_DOSSIER_ID)
|
||||
.hexColor("#ffe187")
|
||||
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||
.build()));
|
||||
|
||||
mockDictionaryCalls(null);
|
||||
|
||||
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,55 @@
|
||||
package drools
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
|
||||
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.LinkedList;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.Collection;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.Optional;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.component.Component;
|
||||
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
|
||||
global ComponentCreationService componentCreationService
|
||||
|
||||
//------------------------------------ queries ------------------------------------
|
||||
|
||||
query "getFileAttributes"
|
||||
$fileAttribute: FileAttribute()
|
||||
end
|
||||
|
||||
query "getComponents"
|
||||
$component: Component()
|
||||
end
|
||||
|
||||
//------------------------------------ Guideline mapping object ------------------------------------
|
||||
|
||||
declare GuidelineMapping
|
||||
number: String
|
||||
year: String
|
||||
guideline: String
|
||||
end
|
||||
|
||||
//------------------------------------ Test Components rules ------------------------------------
|
||||
|
||||
rule "AdvTableExtr.2.3: Cells containing dose for survived males"
|
||||
when
|
||||
$tableValues: List() from collect (Entity(type == "test"))
|
||||
then
|
||||
componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues);
|
||||
end
|
||||
@ -0,0 +1,401 @@
|
||||
package drools
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
|
||||
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.LinkedList;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.Collection;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.Optional;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.*;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
|
||||
global Document document
|
||||
global EntityCreationService entityCreationService
|
||||
global ManualChangesApplicationService manualChangesApplicationService
|
||||
global Dictionary dictionary
|
||||
|
||||
//------------------------------------ queries ------------------------------------
|
||||
|
||||
query "getFileAttributes"
|
||||
$fileAttribute: FileAttribute()
|
||||
end
|
||||
|
||||
//------------------------------------ Local dictionary search rules ------------------------------------
|
||||
|
||||
// Rule unit: LocalDictionarySearch.0
|
||||
rule "LDS.0.0: Run local dictionary search"
|
||||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||||
salience -999
|
||||
when
|
||||
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
|
||||
then
|
||||
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
|
||||
.forEach(entity -> {
|
||||
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
|
||||
entity.addMatchedRules(matchedRules);
|
||||
});
|
||||
end
|
||||
// --------------------------------------- Your rules below this line --------------------------------------------------
|
||||
|
||||
rule "TAB.0.0: Study Type File Attribute"
|
||||
when
|
||||
not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
|
||||
$section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
|
||||
&& containsAnyString("OECD", "EPA", "OPPTS"))
|
||||
then
|
||||
RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
|
||||
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
|
||||
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
|
||||
.forEach(fileAttribute -> insert(fileAttribute));
|
||||
RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
|
||||
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
|
||||
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
|
||||
.forEach(fileAttribute -> insert(fileAttribute));
|
||||
end
|
||||
|
||||
rule "TAB.0.1: Guidelines"
|
||||
when
|
||||
$section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
|
||||
then
|
||||
entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
|
||||
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
|
||||
entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
|
||||
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
|
||||
entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
|
||||
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
|
||||
entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
|
||||
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
|
||||
entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
|
||||
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
|
||||
end
|
||||
|
||||
|
||||
rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
|
||||
when
|
||||
$section: Section(containsString("Maximum occurrence"))
|
||||
$table: Table() from $section.streamChildren().toList()
|
||||
TableCell(containsWordIgnoreCase("water"), $row: row) from $table.streamTableCells().toList()
|
||||
$test: TableCell($row == row) from $table.streamTableCells().toList()
|
||||
then
|
||||
System.out.println("AAAA: " + $test);
|
||||
entityCreationService.bySemanticNode($test, "test", EntityType.ENTITY)
|
||||
.ifPresent(entity -> entity.apply("TAB.6.0", "Some test stuff"));
|
||||
end
|
||||
|
||||
|
||||
//------------------------------------ Manual redaction rules ------------------------------------
|
||||
|
||||
// Rule unit: MAN.0
|
||||
rule "MAN.0.0: Apply manual resize redaction"
|
||||
salience 128
|
||||
when
|
||||
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
||||
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
|
||||
$entityToBeResized: TextEntity(matchesAnnotationId($id))
|
||||
then
|
||||
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
|
||||
retract($resizeRedaction);
|
||||
update($entityToBeResized);
|
||||
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
|
||||
end
|
||||
|
||||
rule "MAN.0.1: Apply manual resize redaction"
|
||||
salience 128
|
||||
when
|
||||
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
||||
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
|
||||
$imageToBeResized: Image(id == $id)
|
||||
then
|
||||
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
|
||||
retract($resizeRedaction);
|
||||
update($imageToBeResized);
|
||||
update($imageToBeResized.getParent());
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: MAN.1
|
||||
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
|
||||
salience 128
|
||||
when
|
||||
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
|
||||
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
|
||||
then
|
||||
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
|
||||
update($entityToBeRemoved);
|
||||
retract($idRemoval);
|
||||
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
|
||||
end
|
||||
|
||||
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
|
||||
salience 128
|
||||
when
|
||||
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
|
||||
$imageEntityToBeRemoved: Image($id == id)
|
||||
then
|
||||
$imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
|
||||
update($imageEntityToBeRemoved);
|
||||
retract($idRemoval);
|
||||
update($imageEntityToBeRemoved.getParent());
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: MAN.2
|
||||
rule "MAN.2.0: Apply force redaction"
|
||||
salience 128
|
||||
when
|
||||
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
|
||||
$entityToForce: TextEntity(matchesAnnotationId($id))
|
||||
then
|
||||
$entityToForce.getManualOverwrite().addChange($force);
|
||||
update($entityToForce);
|
||||
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
|
||||
retract($force);
|
||||
end
|
||||
|
||||
rule "MAN.2.1: Apply force redaction to images"
|
||||
salience 128
|
||||
when
|
||||
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
|
||||
$imageToForce: Image(id == $id)
|
||||
then
|
||||
$imageToForce.getManualOverwrite().addChange($force);
|
||||
update($imageToForce);
|
||||
update($imageToForce.getParent());
|
||||
retract($force);
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: MAN.3
|
||||
rule "MAN.3.0: Apply entity recategorization"
|
||||
salience 128
|
||||
when
|
||||
$recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
||||
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
||||
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
|
||||
then
|
||||
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
|
||||
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
|
||||
retract($recategorization);
|
||||
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
|
||||
retract($entityToBeRecategorized);
|
||||
end
|
||||
|
||||
rule "MAN.3.1: Apply entity recategorization of same type"
|
||||
salience 128
|
||||
when
|
||||
$recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
||||
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
||||
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
|
||||
then
|
||||
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
|
||||
retract($recategorization);
|
||||
end
|
||||
|
||||
rule "MAN.3.2: Apply image recategorization"
|
||||
salience 128
|
||||
when
|
||||
$recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
||||
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
||||
$imageToBeRecategorized: Image($id == id)
|
||||
then
|
||||
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
|
||||
update($imageToBeRecategorized);
|
||||
update($imageToBeRecategorized.getParent());
|
||||
retract($recategorization);
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: MAN.4
|
||||
rule "MAN.4.0: Apply legal basis change"
|
||||
salience 128
|
||||
when
|
||||
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
|
||||
$imageToBeRecategorized: Image($id == id)
|
||||
then
|
||||
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
|
||||
end
|
||||
|
||||
rule "MAN.4.1: Apply legal basis change"
|
||||
salience 128
|
||||
when
|
||||
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
|
||||
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
|
||||
then
|
||||
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
|
||||
end
|
||||
|
||||
|
||||
//------------------------------------ Entity merging rules ------------------------------------
|
||||
|
||||
// Rule unit: X.0
|
||||
rule "X.0.0: Remove Entity contained by Entity of same type"
|
||||
salience 65
|
||||
when
|
||||
$larger: TextEntity($type: type, $entityType: entityType, active())
|
||||
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
|
||||
then
|
||||
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
|
||||
retract($contained);
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: X.1
|
||||
rule "X.1.0: Merge intersecting Entities of same type"
|
||||
salience 64
|
||||
when
|
||||
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
|
||||
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
|
||||
then
|
||||
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
|
||||
$first.remove("X.1.0", "merge intersecting Entities of same type");
|
||||
$second.remove("X.1.0", "merge intersecting Entities of same type");
|
||||
retract($first);
|
||||
retract($second);
|
||||
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: X.2
|
||||
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
||||
salience 64
|
||||
when
|
||||
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
|
||||
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
|
||||
then
|
||||
$entity.getIntersectingNodes().forEach(node -> update(node));
|
||||
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
|
||||
retract($entity)
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: X.3
|
||||
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
|
||||
salience 64
|
||||
when
|
||||
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
|
||||
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
|
||||
then
|
||||
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
|
||||
retract($recommendation);
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: X.4
|
||||
rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
|
||||
salience 256
|
||||
when
|
||||
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
|
||||
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
|
||||
then
|
||||
$entity.addEngines($recommendation.getEngines());
|
||||
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
|
||||
retract($recommendation);
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: X.5
|
||||
rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
|
||||
salience 256
|
||||
when
|
||||
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
|
||||
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
|
||||
then
|
||||
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
|
||||
retract($recommendation);
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: X.6
|
||||
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
|
||||
salience 32
|
||||
when
|
||||
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
|
||||
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
|
||||
then
|
||||
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
|
||||
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
|
||||
retract($lowerRank);
|
||||
end
|
||||
|
||||
|
||||
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
|
||||
salience 32
|
||||
when
|
||||
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
|
||||
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
|
||||
then
|
||||
$higherRank.getIntersectingNodes().forEach(node -> update(node));
|
||||
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
|
||||
retract($higherRank);
|
||||
end
|
||||
|
||||
|
||||
// Rule unit: X.7
|
||||
rule "X.7.0: remove all images"
|
||||
salience 512
|
||||
when
|
||||
$image: Image(imageType != ImageType.OCR, !hasManualChanges())
|
||||
then
|
||||
$image.remove("X.7.0", "remove all images");
|
||||
retract($image);
|
||||
end
|
||||
|
||||
|
||||
//------------------------------------ File attributes rules ------------------------------------
|
||||
|
||||
// Rule unit: FA.1
|
||||
rule "FA.1.0: Remove duplicate FileAttributes"
|
||||
salience 64
|
||||
when
|
||||
$fileAttribute: FileAttribute($label: label, $value: value)
|
||||
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
|
||||
then
|
||||
retract($duplicate);
|
||||
end
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user