Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-8610' into 'master'

Browse Source 
RED-8610: Dictionary remove on dossier level should be displayed as skipped

Closes RED-8610

See merge request redactmanager/redaction-service!335
This commit is contained in:
Kilian Schüttler 2024-03-22 10:19:59 +01:00
commit 7303a7579b
25 changed files with 243 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.96.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.377.0"
val persistenceServiceVersion = "2.379.0"
val springBootStarterVersion = "3.1.5"
configurations {

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java
View File

@ -166,6 +166,7 @@ public final class MigrationEntity {
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
case RECOMMENDATION -> EntryType.RECOMMENDATION;
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
default -> EntryType.FALSE_POSITIVE;
};
}

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java
View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
@ -11,11 +10,11 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@Data
@AllArgsConstructor
@Slf4j
public class DictionaryModel implements Serializable {
private final String type;
@ -29,6 +28,7 @@ public class DictionaryModel implements Serializable {
private final Set<DictionaryEntryModel> falseRecommendations;
private transient SearchImplementation entriesSearch;
private transient SearchImplementation deletionEntriesSearch;
private transient SearchImplementation falsePositiveSearch;
private transient SearchImplementation falseRecommendationsSearch;
@ -56,20 +56,6 @@ public class DictionaryModel implements Serializable {
this.entries = entries;
this.falsePositives = falsePositives;
this.falseRecommendations = falseRecommendations;
this.entriesSearch = new SearchImplementation(this.entries.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntryModel::getValue)
.collect(Collectors.toList()), caseInsensitive);
this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntryModel::getValue)
.collect(Collectors.toList()), caseInsensitive);
this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive);
}
@ -94,6 +80,18 @@ public class DictionaryModel implements Serializable {
}
public SearchImplementation getDeletionEntriesSearch() {
if (deletionEntriesSearch == null) {
this.deletionEntriesSearch = new SearchImplementation(this.entries.stream()
.filter(DictionaryEntry::isDeleted)
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive);
}
return deletionEntriesSearch;
}
public SearchImplementation getFalsePositiveSearch() {
if (falsePositiveSearch == null) {

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java
View File

@ -5,5 +5,6 @@ public enum EntityType {
HINT,
RECOMMENDATION,
FALSE_POSITIVE,
FALSE_RECOMMENDATION
FALSE_RECOMMENDATION,
DICTIONARY_REMOVAL
}

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
View File

@ -40,7 +40,7 @@ public class TextEntity implements IEntity {
TextRange textRange;
@Builder.Default
List<TextRange> duplicateTextRanges = new ArrayList<>();
String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
String type; // TODO: make final once ManualChangesApplicationService::recategorize is deleted
final EntityType entityType;
@Builder.Default
@ -182,25 +182,37 @@ public class TextEntity implements IEntity {
public boolean containedBy(TextEntity textEntity) {
return this.textRange.containedBy(textEntity.getTextRange())
|| duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange))
|| duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::containedBy));
return this.textRange.containedBy(textEntity.getTextRange()) //
|| duplicateTextRanges.stream()
.anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange)) //
|| duplicateTextRanges.stream()
.anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
.stream()
.anyMatch(duplicateTextRange::containedBy));
}
public boolean contains(TextEntity textEntity) {
return this.textRange.contains(textEntity.getTextRange())
|| duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange))
|| duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::contains));
return this.textRange.contains(textEntity.getTextRange()) //
|| duplicateTextRanges.stream()
.anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange)) //
|| duplicateTextRanges.stream()
.anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
.stream()
.anyMatch(duplicateTextRange::contains));
}
public boolean intersects(TextEntity textEntity) {
return this.textRange.intersects(textEntity.getTextRange())
|| duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange))
|| duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::intersects));
return this.textRange.intersects(textEntity.getTextRange()) //
|| duplicateTextRanges.stream()
.anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange)) //
|| duplicateTextRanges.stream()
.anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
.stream()
.anyMatch(duplicateTextRange::intersects));
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -6,10 +6,11 @@ import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@ -38,10 +39,13 @@ public class DictionarySearchService {
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
for (var model : dictionary.getDictionaryModels()) {
for (DictionaryModel model : dictionary.getDictionaryModels()) {
bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
if (model.isDossierDictionary()) {
bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
}
}
}
@ -52,11 +56,12 @@ public class DictionarySearchService {
SemanticNode node,
boolean isDossierDictionaryEntry) {
Set<Engine> engines = Set.of(Engine.DICTIONARY);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.stream()
.filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY))
.forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
.ifPresent(entity -> {
entity.setDictionaryEntry(true);
entity.setDossierDictionaryEntry(isDossierDictionaryEntry);

27
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java
View File

@ -1,9 +1,9 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.awt.Color;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
@ -368,30 +368,23 @@ public class DictionaryService {
@Observed(name = "DictionaryService", contextualName = "deep-copy-dictionary")
public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
List<DictionaryModel> mergedDictionaries;
List<DictionaryModel> mergedDictionaries = new LinkedList<>();
var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
var dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
List<DictionaryModel> dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
// merge dictionaries if they have same names
long dossierDictionaryVersion = -1;
if (dossierDictionaryExists(dossierId)) {
var dossierRepresentation = getDossierDictionary(dossierId);
var dossierDictionaries = dossierRepresentation.getDictionary();
mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries),
convertDictionaryModel(dossierDictionaries)));
dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
} else {
mergedDictionaries = new ArrayList<>();
dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
}
// add dossier
DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId);
List<DictionaryModel> dossierDictionaries = dossierRepresentation.getDictionary();
dossierDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
return new Dictionary(mergedDictionaries.stream()
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.collect(Collectors.toList()),
DictionaryVersion.builder()
.dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
.dossierVersion(dossierDictionaryVersion)
.dossierVersion(dossierRepresentation.getDictionaryVersion())
.build());
}

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -52,9 +52,11 @@ public class EntityLogCreatorService {
RedactionStorageService redactionStorageService;
private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {
return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION));
return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) //
|| textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION) //
|| textEntity.getEntityType().equals(EntityType.DICTIONARY_REMOVAL));
}
@ -145,7 +147,7 @@ public class EntityLogCreatorService {
document.getEntities()
.stream()
.filter(entity -> !entity.getValue().isEmpty())
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
.filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
document.streamAllImages()
@ -341,6 +343,7 @@ public class EntityLogCreatorService {
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
case RECOMMENDATION -> EntryType.RECOMMENDATION;
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
case DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE;
};
}

11
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -233,18 +233,13 @@ public class DictionaryServiceTest {
dictionaryService.updateDictionary("dtId", "dossierId");
var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId");
assertThat(dict.getDictionaryModels().size()).isEqualTo(1);
assertThat(dict.getDictionaryModels().size()).isEqualTo(2);
var dictModel = dict.getDictionaryModels()
.get(0);
assertThat(dictModel.getType()).isEqualTo(type);
assertThat(dictModel.getEntries().size()).isEqualTo(4);
assertThat(dictModel.getEntries().size()).isEqualTo(3);
dictModel.getEntries()
.forEach(entry -> {
switch (entry.getValue()) {
case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId());
case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId());
}
});
.forEach(entry -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId()));
}
}

14
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -1335,6 +1335,20 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

30
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -1037,6 +1037,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.4
rule "PII.4.0: Redact line after contact information keywords"
when
@ -1340,8 +1341,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.12
rule "PII.12.1: Expand PII entities with salutation prefix"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1351,6 +1350,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix"
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.13
rule "PII.13.0: Add recommendation for PII after Contact Person"
when
@ -1542,6 +1542,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
$logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: ETC.9
rule "ETC.9.0: Redact skipped impurities"
when
@ -1559,6 +1560,7 @@ rule "ETC.9.1: Redact impurities"
$skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
end
// Rule unit: ETC.10
rule "ETC.10.0: Redact Product Composition Information"
when
@ -1567,6 +1569,7 @@ rule "ETC.10.0: Redact Product Composition Information"
$compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
end
// Rule unit: ETC.11
rule "ETC.11.0: Recommend first line in table cell with name and address of owner"
when
@ -1596,6 +1599,7 @@ rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)"
$dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
@ -1641,7 +1645,8 @@ rule "AI.3.0: Recommend authors from AI as PII"
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -1761,7 +1766,6 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization);
end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
@ -1770,6 +1774,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
@ -1930,6 +1935,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
retract($other);
end
// Rule unit: X.9
rule "X.9.0: Merge mostly contained signatures"
when
@ -1940,6 +1946,7 @@ rule "X.9.0: Merge mostly contained signatures"
$signature.addEngine(LayoutEngine.AI);
end
// Rule unit: X.10
rule "X.10.0: remove false positives of ai"
when
@ -1949,6 +1956,21 @@ rule "X.10.0: remove false positives of ai"
$aiSignature.remove("X.10.0", "Removed because false positive");
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

14
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1437,6 +1437,20 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

17
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -296,6 +296,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
});
end
// Rule unit: CBI.23
rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when
@ -315,6 +316,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
.forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
//------------------------------------ PII rules ------------------------------------
// Rule unit: PII.0
@ -504,6 +506,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
.forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.10
rule "PII.10.0: Redact study director abbreviation"
when
@ -947,6 +950,20 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

14
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -278,6 +278,20 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0

16
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -274,7 +274,6 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
@ -602,7 +601,6 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
//------------------------------------ PII rules ------------------------------------
// Rule unit: PII.0
rule "PII.0.0: Redact all PII"
when
@ -1425,6 +1423,20 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

14
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -402,6 +402,20 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

14
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -523,6 +523,20 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

14
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -439,6 +439,20 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

6
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java
View File

@ -33,11 +33,7 @@ public class RuleFileMigrator {
rulesToAdd.forEach(ruleFileBluePrint::addRule);
}
RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(),
combinedBluePrint.globals(),
combinedBluePrint.queries(),
ruleFileBluePrint.ruleClasses());
String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint);
String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
String migratedFilePath = ruleFile.getAbsolutePath();
try (var out = new FileOutputStream(migratedFilePath)) {
out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));

1
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java
View File

@ -17,6 +17,7 @@ public class RuleType {
"ETC", "Other",//
"MAN", "Manual changes",//
"X", "Entity merging",//
"DICT", "Dictionary merging",//
"FA", "File attributes",//
"LDS", "Local dictionary search",//
"TAB", "Table extraction",//

15
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -1949,6 +1949,21 @@ rule "X.10.0: remove false positives of ai"
$aiSignature.remove("X.10.0", "Removed because false positive");
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

13
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1582,6 +1582,19 @@ rule "X.10.0: remove false positives of ai"
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------

1
redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt
View File

@ -8,5 +8,6 @@ X.5.0
X.5.1
X.6.*
X.8.*
DICT.*.*
FA.*.*
LDS.*.*

1
redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt
View File

@ -7,5 +7,6 @@ X.5.0
X.5.1
X.7.0
X.8.*
DICT.*.*
FA.*.*
LDS.*.*

1
redaction-service-v1/rules-management/src/main/resources/order_template.txt
View File

@ -9,5 +9,6 @@ ETC
AI
MAN
X
DICT
FA
LDS