Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009 - Document Tree Structure

Browse Source 
* fix Ner Adapter
This commit is contained in:
Kilian Schuettler 2023-05-03 02:30:04 +02:00
parent 06c4162018
commit 1bcb43e50e
1 changed files with 7 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java
View File

@ -2,11 +2,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.adapter;
import java.util.Collection; import java.util.Collection;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -88,24 +86,16 @@ public class NerEntitiesAdapter {
} }
} }
List<Boundary> boundaries = entityClusters.stream().filter(cluster -> cluster.size() >= MIN_PART_MATCHES).map(this::toContainingBoundary).distinct().toList(); return entityClusters.stream()
.filter(cluster -> cluster.size() >= MIN_PART_MATCHES)
return boundaries.stream() .map(this::toContainingBoundary)
.map(boundary -> removeTrailingSeparator(boundary, textBlock)) .distinct()
.filter(boundary -> SeparatorUtils.boundaryIsSurroundedBySeparators(textBlock, boundary))
.map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address")) .map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address"))
.toList(); .toList();
} }
private Boundary removeTrailingSeparator(Boundary boundary, TextBlock textBlock) {
if (SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1))) {
return new Boundary(boundary.start(), boundary.end());
}
return boundary;
}
private static boolean isDuplicate(List<EntityRecognitionEntity> currentCluster, EntityRecognitionEntity entity) { private static boolean isDuplicate(List<EntityRecognitionEntity> currentCluster, EntityRecognitionEntity entity) {
return ALLOW_DUPLICATES || currentCluster.stream().anyMatch(e -> e.getType().equals(entity.getType())); return ALLOW_DUPLICATES || currentCluster.stream().anyMatch(e -> e.getType().equals(entity.getType()));
@ -121,40 +111,8 @@ public class NerEntitiesAdapter {
public Boundary toContainingBoundary(Collection<EntityRecognitionEntity> entityRecognitionEntities) { public Boundary toContainingBoundary(Collection<EntityRecognitionEntity> entityRecognitionEntities) {
int minStart = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).min().orElseThrow(IllegalArgumentException::new); int minStart = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).min().orElseThrow(IllegalArgumentException::new);
int maxEnd = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).max().orElseThrow(IllegalArgumentException::new); int maxEnd = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getEndOffset).max().orElseThrow(IllegalArgumentException::new);
return new Boundary(minStart, maxEnd); return new Boundary(minStart, maxEnd - 1);
}
public Set<EntityRecognitionEntity> findFollowingEntitiesOfTypeCBIAddressPart(EntityRecognitionEntity entityOfCBIAddressEssentialType,
Set<EntityRecognitionEntity> entitiesOfTypeCBIAddressPart) {
Set<EntityRecognitionEntity> concurrentParts = new HashSet<>();
concurrentParts.add(entityOfCBIAddressEssentialType);
int currentNumberOfConcurrentParts;
do {
currentNumberOfConcurrentParts = concurrentParts.size();
concurrentParts.addAll(entitiesOfTypeCBIAddressPart.stream()
.filter(entity -> entity.getStartOffset() >= toContainingBoundary(concurrentParts).end())
.filter(entity -> offsetBetweenPartsIsLowerThanMax(toContainingBoundary(concurrentParts), entity))
.filter(entity -> ALLOW_DUPLICATES || entityTypeInAlreadyFound(concurrentParts, entity.getType()))
.collect(Collectors.toSet()));
} while (currentNumberOfConcurrentParts != concurrentParts.size());
return concurrentParts;
}
private static boolean entityTypeInAlreadyFound(Set<EntityRecognitionEntity> alreadyFound, String type) {
return !alreadyFound.stream().map(EntityRecognitionEntity::getType).collect(Collectors.toSet()).contains(type);
}
private static boolean offsetBetweenPartsIsLowerThanMax(Boundary currentBoundary, EntityRecognitionEntity entity) {
return Math.abs(entity.getStartOffset() - currentBoundary.end()) < MAX_DISTANCE_BETWEEN_PARTS;
} }