Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RSS-118: Refactored method in Section for adding AI entries

Browse Source
This commit is contained in:
Philipp Schramm 2022-10-27 14:15:42 +02:00
parent 1e6e5e2154
commit b4f079c3c2
1 changed files with 61 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -80,39 +80,29 @@ public class Section {
private ManualRedactions manualRedactions; private ManualRedactions manualRedactions;
@Deprecated
@SuppressWarnings("unused") @SuppressWarnings("unused")
@ThenAction @ThenAction
public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) { public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
redactOrRecommendAiEntities(type, asType, false, 0, null, null);
}
Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
Set<Entity> found = EntitySearchUtils.findEntities(searchText,
new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)),
dictionary.getType(asType),
new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
Set<Entity> finalResult = new HashSet<>(); @SuppressWarnings("unused")
@ThenAction
public void recommendAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
redactOrRecommendAiEntities(type, asType, false, 0, null, null);
}
// Only keep Entities with correct offsets from AI Service. @SuppressWarnings("unused")
for (Entity current : found) { @ThenAction
boolean foundSameOffsets = false; public void redactAiEntities(@Argument(ArgumentType.TYPE) String type,
for (Entity entity : entitiesOfType) { @Argument(ArgumentType.TYPE) String asType,
if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) { @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
foundSameOffsets = true; @Argument(ArgumentType.STRING) String reason,
break; @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
}
}
if (foundSameOffsets) {
finalResult.add(current);
}
}
var nonOverlapping = EntitySearchUtils.findNonOverlappingMatchEntities(entities, finalResult); redactOrRecommendAiEntities(type, asType, true, ruleNumber, reason, legalBasis);
EntitySearchUtils.addEntitiesWithHigherRank(entities, nonOverlapping, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
nerEntities.removeAll(entitiesOfType);
} }
@ -1395,6 +1385,51 @@ public class Section {
} }
private void redactOrRecommendAiEntities(String type, String asType, boolean redact, int ruleNumber, String reason, String legalBasis) {
Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
Set<Entity> found = EntitySearchUtils.findEntities(searchText,
new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)),
dictionary.getType(asType),
new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
Set<Entity> finalResult = new HashSet<>();
// Only keep Entities with correct offsets from AI Service.
for (Entity current : found) {
boolean foundSameOffsets = false;
for (Entity entity : entitiesOfType) {
if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) {
foundSameOffsets = true;
break;
}
}
if (foundSameOffsets) {
finalResult.add(current);
}
}
var nonOverlapping = EntitySearchUtils.findNonOverlappingMatchEntities(entities, finalResult);
if (redact) {
nonOverlapping.forEach(entity -> {
entity.setRedaction(true);
entity.setEntityType(EntityType.ENTITY);
entity.setMatchedRule(ruleNumber);
entity.setRedactionReason(reason);
entity.setLegalBasis(legalBasis);
});
}
EntitySearchUtils.addEntitiesWithHigherRank(entities, nonOverlapping, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
nerEntities.removeAll(entitiesOfType);
}
@Retention(RetentionPolicy.RUNTIME) @Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD) @Target(ElementType.METHOD)
public @interface WhenCondition { public @interface WhenCondition {