DM-285: update component rules

This commit is contained in:
Kilian Schüttler 2023-09-26 20:05:22 +02:00 committed by Kresnadi Budisantoso
parent 0053fc9fac
commit 8d07de8630
7 changed files with 146 additions and 113 deletions

View File

@ -80,8 +80,7 @@ public class Entity {
.legalBasis(e.getLegalBasis())
.imported(e.isImported())
.section(e.getSection())
.color(e.getColor())
.positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
.color(e.getColor()).positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
.textBefore(e.getTextBefore())
.textAfter(e.getTextAfter())
.startOffset(e.getStartOffset())

View File

@ -23,12 +23,12 @@ public class EntityChangeLogService {
@Timed("redactmanager_computeChanges")
public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> currentEntityLogEntries, int analysisNumber) {
public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
boolean hasChanges = false;
var now = OffsetDateTime.now();
for (EntityLogEntry entityLogEntry : currentEntityLogEntries) {
for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
var optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
if (optionalPreviousEntity.isEmpty()) {
hasChanges = true;
@ -44,7 +44,7 @@ public class EntityChangeLogService {
}
}
}
Set<String> existingIds = currentEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream().filter(entry -> !existingIds.contains(entry.getId())).toList();
removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));

View File

@ -91,13 +91,13 @@ public class EntityLogCreatorService {
DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
List<EntityLogEntry> previousEntries = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> (entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> entriesFromReanalysedSections = previousEntityLog.getEntityLogEntry()
.stream().filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
.equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
.toList();
previousEntityLog.getEntityLogEntry().removeAll(previousEntries);
boolean hasChanges = entityChangeLogService.computeChanges(previousEntries, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
previousEntityLog.getEntityLogEntry().removeAll(entriesFromReanalysedSections);
boolean hasChanges = entityChangeLogService.computeChanges(entriesFromReanalysedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
@ -106,7 +106,6 @@ public class EntityLogCreatorService {
false);
previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages());

View File

@ -4,6 +4,7 @@ import java.text.BreakIterator;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@ -12,6 +13,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.kie.api.runtime.KieSession;
@ -51,14 +53,42 @@ public class ComponentCreationService {
}
/**
* Joins entity values, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joining(String ruleIdentifier, String name, Collection<Entity> entities) {
joining(ruleIdentifier, name, entities, ", ");
}
/**
* Joins entity values, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
String valueDescription = String.format("Joining all values with '%s'", delimiter);
String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
create(ruleIdentifier, name, value, valueDescription, entities);
}
/**
* Creates a new component with the given parameters and inserts it into the kieSession.
*
* @param ruleIdentifier The rule identifier for the component.
* @param name The name of the component.
* @param value The value of the component.
* @param ruleIdentifier The rule identifier for the component.
* @param name The name of the component.
* @param value The value of the component.
* @param valueDescription The description of the value.
* @param references A collection of Entity objects that the component references.
* @param references A collection of Entity objects that the component references.
*/
public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection<Entity> references) {
@ -70,40 +100,12 @@ public class ComponentCreationService {
}
/**
* Joins entity values, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joining(String ruleIdentifier, String name, Collection<Entity> entities) {
joining(ruleIdentifier, name, entities, ", ");
}
/**
* Joins entity values, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
String valueDescription = String.format("Joining all values with '%s'", delimiter);
String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
create(ruleIdentifier, name, value, valueDescription, entities);
}
/**
* Joins entity values from the first section entities appear in, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
@ -115,8 +117,8 @@ public class ComponentCreationService {
* Joins unique entity values from the first section entities appear in, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
@ -140,8 +142,8 @@ public class ComponentCreationService {
* Joins unique entity values from the first section entities appear in, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
@ -153,8 +155,8 @@ public class ComponentCreationService {
* Joins entity values from the first section entities appear in, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
@ -167,9 +169,9 @@ public class ComponentCreationService {
* Joins all unique values from a collection of entities into a single string using a specified delimiter and creates a component from the result.
*
* @param ruleIdentifier the identifier of the rule
* @param name the name of the joining operation
* @param entities the collection of entities
* @param delimiter the delimiter to use for joining the values
* @param name the name of the joining operation
* @param entities the collection of entities
* @param delimiter the delimiter to use for joining the values
*/
public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
@ -196,8 +198,8 @@ public class ComponentCreationService {
* Joins entity values from the section with the longest combined entity values only, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
* @param name The name of the entity.
* @param entities The collection of entities to process.
*/
public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
@ -210,8 +212,7 @@ public class ComponentCreationService {
var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
.stream()
.sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()).map(Map.Entry::getKey)
.stream().sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()).map(Map.Entry::getKey)
.findFirst();
if (longestSection.isEmpty()) {
@ -245,9 +246,9 @@ public class ComponentCreationService {
* Joins unique entity values from the section with the longest combined entity values only, and creates a component from the result.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities The collection of entities to process.
* @param delimiter The string delimiter to join the entities.
* @param name The name of the entity.
* @param entities The collection of entities to process.
* @param delimiter The string delimiter to join the entities.
*/
public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
@ -261,8 +262,8 @@ public class ComponentCreationService {
* and creates a component from the result.
*
* @param ruleIdentifier The unique identifier of the rule being applied.
* @param name The name to which the joined result will be set.
* @param entities The collection of entities whose names will be joined.
* @param name The name to which the joined result will be set.
* @param entities The collection of entities whose names will be joined.
*/
public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities) {
@ -288,8 +289,8 @@ public class ComponentCreationService {
* Creates a component for each sentence in the collection of entities.
*
* @param ruleIdentifier The identifier of the rule.
* @param name The name of the entity.
* @param entities A collection of Entity objects.
* @param name The name of the entity.
* @param entities A collection of Entity objects.
*/
public void asSentences(String ruleIdentifier, String name, Collection<Entity> entities) {
@ -316,19 +317,24 @@ public class ComponentCreationService {
* The component is built using the provided parameters and inserted into the knowledge session.
* The reference is added to the referencedEntities list.
*
* @param ruleIdentifier The identifier of the rule for the component.
* @param name The name of the component.
* @param value The value of the component.
* @param ruleIdentifier The identifier of the rule for the component.
* @param name The name of the component.
* @param value The value of the component.
* @param valueDescription The description of the value.
* @param reference The reference entity for the component.
* @param reference The reference entity for the component.
*/
public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) {
referencedEntities.add(reference);
List<Entity> referenceList = new LinkedList<>();
referenceList.add(reference);
kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value)
.valueDescription(valueDescription).references(referenceList).build());
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.name(name)
.value(value)
.valueDescription(valueDescription)
.references(referenceList)
.build());
}
@ -336,7 +342,7 @@ public class ComponentCreationService {
* Creates components for unmapped entities.
*
* @param ruleIdentifier The identifier of the rule being applied.
* @param entities The collection of entities to create components for.
* @param entities The collection of entities to create components for.
*/
public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
@ -350,8 +356,8 @@ public class ComponentCreationService {
* Converts entity values to the 'dd/MM/yyyy' format and joins them with ', '. If the value could not be parsed as a date, it will be created as is.
*
* @param ruleIdentifier the identifier of the rule
* @param name the name of the entity
* @param entities the collection of entities
* @param name the name of the entity
* @param entities the collection of entities
*/
public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities) {
@ -363,15 +369,32 @@ public class ComponentCreationService {
* Converts entity values to provided format and joins them with ', '. If the value could not be parsed as a date, it will be created as is.
*
* @param ruleIdentifier the identifier of the rule
* @param name the name of the entity
* @param entities the collection of entities
* @param resultFormat the desired format for the converted dates
* @param name the name of the entity
* @param entities the collection of entities
* @param resultFormat the desired format for the converted dates
*/
public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities, String resultFormat) {
String valueDescription = String.format("Convert values of type to %s joined with ', '", resultFormat);
String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
create(ruleIdentifier, name, date, valueDescription, entities);
List<String> unparsedDates = new LinkedList<>();
List<Date> dates = new LinkedList<>();
for (Entity entity : entities) {
String value = entity.getValue();
Optional<Date> optionalDate = DateConverter.parseDate(value);
if (optionalDate.isPresent()) {
dates.add(optionalDate.get());
} else {
unparsedDates.add(value);
}
}
String formattedDateStrings = Stream.concat(//
dates.stream().sorted().map(date -> DateConverter.convertDate(date, resultFormat)), //
unparsedDates.stream())//
.collect(Collectors.joining(", "));
create(ruleIdentifier, name, formattedDateStrings, valueDescription, entities);
}
@ -426,9 +449,9 @@ public class ComponentCreationService {
* Creates a new component with the given rule identifier, name, value, and value description.
* If the component is part of a table, it also takes a list of entities that belong to the same table row.
*
* @param ruleIdentifier the identifier of the rule
* @param name the name of the entity
* @param value the value of the component
* @param ruleIdentifier the identifier of the rule
* @param name the name of the entity
* @param value the value of the component
* @param valueDescription the description of the value
*/
public void create(String ruleIdentifier, String name, String value, String valueDescription) {
@ -441,8 +464,8 @@ public class ComponentCreationService {
* Creates a new component with the given rule identifier, name, and value.
*
* @param ruleIdentifier the identifier of the rule
* @param name the name of the entity
* @param value the value of the component
* @param name the name of the entity
* @param value the value of the component
*/
public void create(String ruleIdentifier, String name, String value) {

View File

@ -16,6 +16,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
@ -43,7 +44,11 @@ public class ComponentDroolsExecutionService {
ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
kieSession.setGlobal("componentCreationService", componentCreationService);
entityLog.getEntityLogEntry().stream().map(entry -> Entity.fromEntityLogEntry(entry, document)).forEach(kieSession::insert);
entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED))
.map(entry -> Entity.fromEntityLogEntry(entry, document))
.forEach(kieSession::insert);
fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
@ -55,7 +60,7 @@ public class ComponentDroolsExecutionService {
completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
} catch (ExecutionException e) {
kieSession.dispose();
if(e.getCause() instanceof TimeoutException){
if (e.getCause() instanceof TimeoutException) {
throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
}
throw new RuntimeException(e);

View File

@ -5,6 +5,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
@ -24,9 +25,8 @@ public class DateConverter {
new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH));
public String convertDate(String dateAsString, String resultFormat) {
public Optional<Date> parseDate(String dateAsString) {
DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
Date date = null;
for (SimpleDateFormat format : formats) {
@ -38,10 +38,16 @@ public class DateConverter {
// ignore, try next...
}
}
if (date == null) {
return dateAsString;
return Optional.empty();
}
return Optional.of(date);
}
public String convertDate(Date date, String resultFormat) {
DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
return resultDateFormat.format(date);
}

View File

@ -51,25 +51,25 @@ rule "StudyTitle.0.0: Study Title"
when
$titleCandidates: List() from collect (Entity(type == "title"))
then
componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!");
componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "");
end
rule "PerformingLaboratory.1.0: Performing Laboratory"
when
$laboratoryName: Entity(type == "laboratory_name")
$laboratoryCountry: Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80)
not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset))
$laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
$laboratoryCountry: Entity(type == "laboratory_country", containingNode == $node)
not Entity(type == "laboratory_country", containingNode == $node, Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset))
then
componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!");
componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!", List.of($laboratoryName, $laboratoryCountry));
end
rule "PerformingLaboratory.2.0: Performing Laboratory"
when
$laboratoryName: Entity(type == "laboratory_name")
not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80)
$laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
not Entity(type == "laboratory_country", containingNode == $node)
then
componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!");
componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!", List.of($laboratoryName));
end
rule "PerformingLaboratory.0.2: Performing Laboratory"
@ -77,7 +77,7 @@ rule "PerformingLaboratory.0.2: Performing Laboratory"
when
not Component(name == "Performing_Laboratory")
then
componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback");
componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "", "fallback");
end
@ -85,7 +85,7 @@ rule "ReportNumber.0.0: Report number"
when
$reportNumberCandidates: List() from collect (Entity(type == "report_number"))
then
componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "No report number found!");
componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "");
end
@ -104,16 +104,8 @@ rule "GLPStudy.1.0: GLP Study"
end
rule "TestGuideline.0.0: no test guideline year found, only oecd number"
when
$guidelineNumber: Entity(type == "oecd_guideline_number")
not Entity(type == "oecd_guideline_year")
then
componentCreationService.create("DefaultComponents.4.0", "Test_Guidelines_1", $guidelineNumber.getValue(), "Only OECD Number found!");
end
rule "TestGuideline.1.0: create mappings"
salience 1
rule "TestGuideline.0.0: create mappings"
salience 2
when
Entity(type == "oecd_guideline_number")
Entity(type == "oecd_guideline_year")
@ -149,20 +141,29 @@ salience 1
insert(new GuidelineMapping("487", "2016", "Nº 487: Micronucleus Human Lymphocytes (2016)"));
end
rule "TestGuideline.1.1: match test guidelines with mappings"
rule "TestGuideline.0.1: match test guidelines with mappings"
salience 1
when
GuidelineMapping($year: year, $number: number, $guideline: guideline)
$guidelineNumber: Entity(type == "oecd_guideline_number", value == $number)
$guidelineYear: Entity(type == "oecd_guideline_year", value == $year)
then
componentCreationService.create(
"DefaultComponents.4.1",
"TestGuideline.0.0",
"Test_Guidelines_1",
$guideline,
"OECD Number and guideline year mapped!"
"OECD Number and guideline year mapped!",
List.of($guidelineNumber, $guidelineYear)
);
end
rule "TestGuideline.1.0: no mapping found"
when
not Component(name == "Test_Guidelines_1")
$guideLine: Entity(type == "oecd_guideline")
then
componentCreationService.create("TestGuideline.2.0", "Test_Guidelines_1", $guideLine.getValue(), "No Mapping for OECD number and year found, using fallback instead!", List.of($guideLine));
end
rule "TestGuideline.2.0: Test Guideline 2"
when