Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-7757: added check for case sensitivity when adding local dictionary entries

Browse Source
This commit is contained in:
Ali Oezyetimoglu 2023-11-28 11:28:50 +01:00
parent 775f43ce03
commit 3c99e015aa
12 changed files with 28 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java
View File

@ -7,6 +7,7 @@ import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -106,12 +107,18 @@ public class Dictionary {
if (StringUtils.isEmpty(value)) { if (StringUtils.isEmpty(value)) {
throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value)); throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
} }
boolean isCaseInsensitive = localAccessMap.get(type).isCaseInsensitive();
Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules); Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
String cleanedValue = value;
if (isCaseInsensitive) {
cleanedValue = cleanedValue.toLowerCase(Locale.US);
}
localAccessMap.get(type) localAccessMap.get(type)
.getLocalEntriesWithMatchedRules() .getLocalEntriesWithMatchedRules()
.merge(value.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet())); .merge(cleanedValue.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
if (alsoAddLastname) { if (alsoAddLastname) {
String lastname = value.split(" ")[0]; String lastname = cleanedValue.split(" ")[0];
localAccessMap.get(type) localAccessMap.get(type)
.getLocalEntriesWithMatchedRules() .getLocalEntriesWithMatchedRules()
.merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet())); .merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java
View File

@ -2,6 +2,8 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
import java.io.Serializable; import java.io.Serializable;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -109,4 +111,11 @@ public class DictionaryModel implements Serializable {
return falseRecommendationsSearch; return falseRecommendationsSearch;
} }
public Set<MatchedRule> getMatchedRulesForLocalDictionaryEntry(String value) {
var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value;
return localEntriesWithMatchedRules.get(cleanedValue);
}
} }

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -746,7 +746,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -1563,7 +1563,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1393,7 +1393,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -219,7 +219,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1144,7 +1144,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -360,7 +360,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -481,7 +481,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -397,7 +397,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -1556,7 +1556,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end

2
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1540,7 +1540,7 @@ rule "LDS.0.0: Run local dictionary search"
then then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> { .forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied())); matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
}); });
end end