Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-10290: Improve SearchImplementation logic for dictionaries

Browse Source 
* cleanup
This commit is contained in:
maverickstuder 2024-10-29 15:17:52 +01:00
parent d6739d4852
commit f1090909cc
3 changed files with 1 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java
View File

@ -26,7 +26,7 @@ public class RedactionServiceSettings {
private boolean llmNerServiceEnabled;
private boolean priorityMode = false;
private boolean priorityMode;
private long firstLevelDictionaryCacheMaximumSize = 1000;

82
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java
View File

@ -1,12 +1,9 @@
package com.iqser.red.service.redaction.v1.server.model.dictionary;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
@ -33,11 +30,6 @@ public class DictionaryModel implements Cloneable {
private final Set<DictionaryEntryModel> falsePositives;
private final Set<DictionaryEntryModel> falseRecommendations;
private SearchImplementation entriesSearch;
private SearchImplementation deletionEntriesSearch;
private SearchImplementation falsePositiveSearch;
private SearchImplementation falseRecommendationsSearch;
private final HashMap<String, Set<MatchedRule>> localEntriesWithMatchedRules = new HashMap<>();
private SearchImplementation localSearch;
@ -91,72 +83,6 @@ public class DictionaryModel implements Cloneable {
}
/**
* Returns the search implementation for non-deleted dictionary entries.
*
* @return The {@link SearchImplementation} for non-deleted dictionary entries.
*/
public SearchImplementation getEntriesSearch() {
if (entriesSearch == null) {
this.entriesSearch = new SearchImplementation(this.entries.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive);
}
return entriesSearch;
}
/**
* Returns the search implementation for deleted dictionary entries.
*
* @return The {@link SearchImplementation} for deleted dictionary entries.
*/
public SearchImplementation getDeletionEntriesSearch() {
if (deletionEntriesSearch == null) {
this.deletionEntriesSearch = new SearchImplementation(this.entries.stream()
.filter(DictionaryEntry::isDeleted)
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive);
}
return deletionEntriesSearch;
}
/**
* Returns the search implementation for non-deleted false positive entries.
*
* @return The {@link SearchImplementation} for non-deleted false positive entries.
*/
public SearchImplementation getFalsePositiveSearch() {
if (falsePositiveSearch == null) {
this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive);
}
return falsePositiveSearch;
}
/**
* Returns the search implementation for non-deleted false recommendation entries.
*
* @return The {@link SearchImplementation} for non-deleted false recommendation entries.
*/
public SearchImplementation getFalseRecommendationsSearch() {
if (falseRecommendationsSearch == null) {
this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive);
}
return falseRecommendationsSearch;
}
/**
@ -173,20 +99,12 @@ public class DictionaryModel implements Cloneable {
}
public SearchImplementation test() {
return this.entriesSearch;
}
@Override
public DictionaryModel clone() {
try {
DictionaryModel cloned = (DictionaryModel) super.clone();
cloned.entriesSearch = null;
cloned.deletionEntriesSearch = null;
cloned.falsePositiveSearch = null;
cloned.falseRecommendationsSearch = null;
cloned.localSearch = null;
return cloned;

40
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -62,44 +62,4 @@ public class DictionarySearchService {
});
}
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries-old")
public void addDictionaryEntitiesOld(Dictionary dictionary, SemanticNode node) {
dictionary.getDictionaryModels()
.forEach(model -> {
bySearchImplementationAsDictionary(model.getEntriesSearch(),
model.getType(),
model.isHint() ? EntityType.HINT : EntityType.ENTITY,
node,
model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
if (model.isDossierDictionary()) {
bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
}
});
}
public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation,
String type,
EntityType entityType,
SemanticNode node,
boolean isDossierDictionaryEntry) {
Set<Engine> engines = isDossierDictionaryEntry ? Set.of(Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
searchImplementation.getBoundaries(node.getTextBlock())
.filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
.ifPresent(entity -> {
entity.setDictionaryEntry(true);
entity.setDossierDictionaryEntry(isDossierDictionaryEntry);
if (entityType.equals(EntityType.DICTIONARY_REMOVAL)) {
entity.ignore("DICT.0.0", "Ignore Dossier Dictionary Entity with DICTIONARY_REMOVAL entity type");
}
}));
}
}