RED-6009 - Document Tree Structure
* moved NER Entity combination to drools
This commit is contained in:
Kilian Schuettler
parent
dce1f3b386
commit
a78a18fe6d
@ -4,7 +4,6 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
@ -12,7 +11,7 @@ import lombok.NoArgsConstructor;
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class NerEntities {
|
||||
public class NerEntitiesModel {
|
||||
|
||||
private Map<Integer, List<EntityRecognitionEntity>> data = new HashMap<>();
|
||||
|
||||
@ -27,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
@ -177,6 +178,12 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
|
||||
|
||||
return byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
|
||||
}
|
||||
|
||||
|
||||
public RedactionEntity byTableCellAsHighlight(TableCellNode tableCellNode, String type, EntityType entityType) {
|
||||
|
||||
RedactionEntity highlightEntity = RedactionEntity.initialEntityNode(new Boundary(tableCellNode.getBoundary().start(), tableCellNode.getBoundary().start()),
|
||||
|
||||
@ -0,0 +1,44 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.adapter;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
|
||||
public class NerEntities {
|
||||
|
||||
List<NerEntity> nerEntityList;
|
||||
|
||||
|
||||
public NerEntities() {
|
||||
|
||||
nerEntityList = new LinkedList<>();
|
||||
}
|
||||
|
||||
|
||||
public boolean hasEntitiesOfType(String type) {
|
||||
|
||||
return nerEntityList.stream().anyMatch(nerEntity -> nerEntity.type.equals(type));
|
||||
}
|
||||
|
||||
|
||||
public Stream<NerEntity> streamEntitiesOfType(String type) {
|
||||
|
||||
return nerEntityList.stream().filter(nerEntity -> nerEntity.type().equals(type));
|
||||
}
|
||||
|
||||
|
||||
public record NerEntity(String value, Boundary boundary, String type) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -10,12 +10,11 @@ import java.util.stream.Stream;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
@ -25,94 +24,129 @@ import lombok.experimental.FieldDefaults;
|
||||
public class NerEntitiesAdapter {
|
||||
|
||||
// imported values from SpaCy model, for explanations look here: https://wiki.knecon.com/display/RES/Address+Recognition
|
||||
static Set<String> CBI_ADDRESS_PARTS = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE");
|
||||
static Set<String> CBI_ADDRESS_ESSENTIAL_PARTS = Set.of("ORG", "STREET", "CITY");
|
||||
static Set<String> CBI_ADDRESS_TYPES_TO_COMBINE = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE");
|
||||
static Set<String> CBI_ADDRESS_ESSENTIAL_TYPES = Set.of("ORG", "STREET", "CITY");
|
||||
|
||||
static int MAX_DISTANCE_BETWEEN_PARTS = 20;
|
||||
static int MIN_PART_MATCHES = 3;
|
||||
static int MIN_PARTS_TO_COMBINE = 3;
|
||||
static boolean ALLOW_DUPLICATES;
|
||||
|
||||
|
||||
public List<EntityRecognitionEntity> validateAndCombine(NerEntities nerEntities, DocumentGraph documentGraph) {
|
||||
/**
|
||||
* Adds the appropriate Offsets to the entities from the NER Service and validates, that the values match.
|
||||
*
|
||||
* @param nerEntitiesModel the Entities just as the NER Service returns them
|
||||
* @param documentGraph the document structure, from which the NER Service found the entities
|
||||
* @return a stream of validated entities
|
||||
*/
|
||||
public List<NerEntities.NerEntity> getValidatedEntities(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
|
||||
|
||||
List<EntityRecognitionEntity> entityRecognitionEntities = getValidatedEntities(nerEntities, documentGraph).toList();
|
||||
|
||||
List<EntityRecognitionEntity> cbiAuthors = entityRecognitionEntities.stream().filter(entity -> entity.getType().equals("CBI_author")).toList();
|
||||
List<EntityRecognitionEntity> cbiAddress = combineToCBIAddressEntities(entityRecognitionEntities, documentGraph.buildTextBlock());
|
||||
|
||||
return Stream.concat(cbiAuthors.stream(), cbiAddress.stream()).toList();
|
||||
return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntitiesModel) //
|
||||
.filter(nerEntityModel -> nerEntityOffsetMatchesDocumentGraphOffsets(nerEntityModel, documentGraph.buildTextBlock()))
|
||||
.map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
|
||||
new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
|
||||
nerEntityModel.getType()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
protected Stream<EntityRecognitionEntity> getValidatedEntities(NerEntities nerEntities, DocumentGraph documentGraph) {
|
||||
/**
|
||||
* This function combines all EntityRecognitionEntities to a list of Boundaries.
|
||||
* Each part must be at most maxDistanceBetweenParts apart from one another.
|
||||
* The first part must be an EntityRecognitionEntity of an essential type.
|
||||
* The resulting list must consist of at least minPartsToCombine parts.
|
||||
*
|
||||
* @param nerEntities already validated entities from the NER Service
|
||||
* @param essentialTypes the combined entities must contain at least one of these types
|
||||
* @param typesToCombine all types which should be used to combine, must contain all essentialTypes
|
||||
* @param maxDistanceBetweenParts all parts used to combine should be at most this value apart in string offset coordinates
|
||||
* @param minPartsToCombine minimum number of parts to combine
|
||||
* @param allowDuplicates allow combining multiple parts of same type
|
||||
* @return A Stream of the combined boundaries
|
||||
*/
|
||||
public Stream<Boundary> combineNerEntities(NerEntities nerEntities,
|
||||
Set<String> essentialTypes,
|
||||
Set<String> typesToCombine,
|
||||
int maxDistanceBetweenParts,
|
||||
int minPartsToCombine,
|
||||
boolean allowDuplicates) {
|
||||
|
||||
return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntities) //
|
||||
.filter(nerEntity -> nerEntityOffsetMatchesDocumentGraphOffsets(nerEntity, documentGraph.buildTextBlock()));
|
||||
}
|
||||
|
||||
|
||||
private List<EntityRecognitionEntity> combineToCBIAddressEntities(List<EntityRecognitionEntity> entityRecognitionEntities, TextBlock textBlock) {
|
||||
|
||||
List<EntityRecognitionEntity> sortedEntities = entityRecognitionEntities.stream()
|
||||
.filter(entity -> CBI_ADDRESS_PARTS.contains(entity.getType()))
|
||||
.sorted(Comparator.comparingInt(EntityRecognitionEntity::getStartOffset))
|
||||
List<NerEntities.NerEntity> sortedEntities = nerEntities.getNerEntityList()
|
||||
.stream()
|
||||
.filter(entity -> typesToCombine.contains(entity.type()))
|
||||
.sorted(Comparator.comparingInt(entity -> entity.boundary().start()))
|
||||
.toList();
|
||||
|
||||
if (sortedEntities.isEmpty()) {
|
||||
return sortedEntities;
|
||||
return Stream.empty();
|
||||
}
|
||||
|
||||
List<List<EntityRecognitionEntity>> entityClusters = new LinkedList<>();
|
||||
List<List<NerEntities.NerEntity>> entityClusters = new LinkedList<>();
|
||||
|
||||
List<EntityRecognitionEntity> startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> CBI_ADDRESS_ESSENTIAL_PARTS.contains(e.getType())).toList();
|
||||
for (EntityRecognitionEntity startEntity : startEntitiesOfEssentialType) {
|
||||
List<EntityRecognitionEntity> currentCluster = new LinkedList<>();
|
||||
List<NerEntities.NerEntity> startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> essentialTypes.contains(e.type())).toList();
|
||||
for (NerEntities.NerEntity startEntity : startEntitiesOfEssentialType) {
|
||||
List<NerEntities.NerEntity> currentCluster = new LinkedList<>();
|
||||
entityClusters.add(currentCluster);
|
||||
int lastEndOffset = startEntity.getStartOffset();
|
||||
int lastEndOffset = startEntity.boundary().end();
|
||||
|
||||
for (EntityRecognitionEntity entity : sortedEntities) {
|
||||
if (entity.getStartOffset() < lastEndOffset) {
|
||||
for (NerEntities.NerEntity entity : sortedEntities) {
|
||||
if (entity.boundary().start() < lastEndOffset) {
|
||||
continue;
|
||||
}
|
||||
if (disctanceIsLargerThanMaxDistance(lastEndOffset, entity) || isDuplicate(currentCluster, entity)) {
|
||||
if (distanceIsLargerThanMaxDistance(lastEndOffset, entity, maxDistanceBetweenParts) || isDuplicate(currentCluster, entity, allowDuplicates)) {
|
||||
currentCluster = new LinkedList<>();
|
||||
entityClusters.add(currentCluster);
|
||||
currentCluster.add(entity);
|
||||
lastEndOffset = entity.getEndOffset();
|
||||
lastEndOffset = entity.boundary().end();
|
||||
} else {
|
||||
currentCluster.add(entity);
|
||||
lastEndOffset = entity.getEndOffset();
|
||||
lastEndOffset = entity.boundary().end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return entityClusters.stream()
|
||||
.filter(cluster -> cluster.size() >= MIN_PART_MATCHES)
|
||||
.map(this::toContainingBoundary)
|
||||
.distinct()
|
||||
.filter(boundary -> SeparatorUtils.boundaryIsSurroundedBySeparators(textBlock, boundary))
|
||||
.map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address"))
|
||||
.toList();
|
||||
return entityClusters.stream().filter(cluster -> cluster.size() >= minPartsToCombine).map(NerEntitiesAdapter::toContainingBoundary).distinct();
|
||||
}
|
||||
|
||||
|
||||
private static boolean isDuplicate(List<EntityRecognitionEntity> currentCluster, EntityRecognitionEntity entity) {
|
||||
/**
|
||||
* Calls combine NerEntities with the following settings.
|
||||
* <p>
|
||||
* essentialTypes Set.of("ORG", "STREET", "CITY")
|
||||
* typesToCombine Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE")
|
||||
* resultType "CBI_address"
|
||||
* maxDistanceBetweenParts 20
|
||||
* minPartsToCombine 3
|
||||
* allowDuplicates false
|
||||
*
|
||||
* @return A Stream of the combined entities of type "CBI_address"
|
||||
*/
|
||||
public Stream<Boundary> combineNerEntitiesToCbiAddressDefaults(NerEntities entityRecognitionEntities) {
|
||||
|
||||
return ALLOW_DUPLICATES || currentCluster.stream().anyMatch(e -> e.getType().equals(entity.getType()));
|
||||
return combineNerEntities(entityRecognitionEntities,
|
||||
CBI_ADDRESS_ESSENTIAL_TYPES,
|
||||
CBI_ADDRESS_TYPES_TO_COMBINE,
|
||||
MAX_DISTANCE_BETWEEN_PARTS,
|
||||
MIN_PARTS_TO_COMBINE,
|
||||
ALLOW_DUPLICATES);
|
||||
}
|
||||
|
||||
|
||||
private static boolean disctanceIsLargerThanMaxDistance(int lastEndOffset, EntityRecognitionEntity entity) {
|
||||
private static boolean isDuplicate(List<NerEntities.NerEntity> currentCluster, NerEntities.NerEntity entity, boolean allowDuplicates) {
|
||||
|
||||
return (entity.getStartOffset() - lastEndOffset) > MAX_DISTANCE_BETWEEN_PARTS;
|
||||
return allowDuplicates || currentCluster.stream().anyMatch(e -> e.type().equals(entity.type()));
|
||||
}
|
||||
|
||||
|
||||
public Boundary toContainingBoundary(Collection<EntityRecognitionEntity> entityRecognitionEntities) {
|
||||
private static boolean distanceIsLargerThanMaxDistance(int lastEndOffset, NerEntities.NerEntity entity, int maxDistance) {
|
||||
|
||||
int minStart = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).min().orElseThrow(IllegalArgumentException::new);
|
||||
int maxEnd = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getEndOffset).max().orElseThrow(IllegalArgumentException::new);
|
||||
return new Boundary(minStart, maxEnd - 1);
|
||||
return (entity.boundary().start() - lastEndOffset) > maxDistance;
|
||||
}
|
||||
|
||||
|
||||
private static Boundary toContainingBoundary(List<NerEntities.NerEntity> nerEntities) {
|
||||
|
||||
return Boundary.merge(nerEntities.stream().map(NerEntities.NerEntity::boundary).toList());
|
||||
}
|
||||
|
||||
|
||||
@ -122,14 +156,14 @@ public class NerEntitiesAdapter {
|
||||
}
|
||||
|
||||
|
||||
private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntities nerEntities) {
|
||||
private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) {
|
||||
|
||||
nerEntities.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> {
|
||||
nerEntitiesModel.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> {
|
||||
int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key);
|
||||
entityRecognitionEntity.setStartOffset(newStartOffset);
|
||||
entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length());
|
||||
}));
|
||||
return nerEntities.getData().values().stream().flatMap(Collection::stream);
|
||||
return nerEntitiesModel.getData().values().stream().flatMap(Collection::stream);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -22,12 +22,13 @@ import org.springframework.stereotype.Service;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
||||
@ -49,6 +50,7 @@ public class DroolsExecutionService {
|
||||
Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
|
||||
EntityCreationService entityCreationService;
|
||||
ManualRedactionApplicationService manualRedactionApplicationService;
|
||||
NerEntitiesAdapter nerEntitiesAdapter;
|
||||
|
||||
|
||||
public KieContainer getKieContainer(String dossierTemplateId) {
|
||||
@ -68,7 +70,7 @@ public class DroolsExecutionService {
|
||||
Dictionary dictionary,
|
||||
List<FileAttribute> fileAttributes,
|
||||
ManualRedactions manualRedactions,
|
||||
List<EntityRecognitionEntity> nerEntities) {
|
||||
NerEntities nerEntities) {
|
||||
|
||||
return executeRules(kieContainer, document, document.streamChildren().toList(), dictionary, fileAttributes, manualRedactions, nerEntities);
|
||||
}
|
||||
@ -81,13 +83,14 @@ public class DroolsExecutionService {
|
||||
Dictionary dictionary,
|
||||
List<FileAttribute> fileAttributes,
|
||||
ManualRedactions manualRedactions,
|
||||
List<EntityRecognitionEntity> nerEntities) {
|
||||
NerEntities nerEntities) {
|
||||
|
||||
KieSession kieSession = kieContainer.newKieSession();
|
||||
kieSession.setGlobal("document", document);
|
||||
kieSession.setGlobal("entityCreationService", entityCreationService);
|
||||
kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
|
||||
kieSession.setGlobal("dictionary", dictionary);
|
||||
kieSession.setGlobal("nerEntitiesAdapter", nerEntitiesAdapter);
|
||||
|
||||
document.getEntities().forEach(kieSession::insert);
|
||||
sectionsToAnalyze.forEach(kieSession::insert);
|
||||
@ -103,7 +106,7 @@ public class DroolsExecutionService {
|
||||
manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
|
||||
}
|
||||
|
||||
nerEntities.forEach(kieSession::insert);
|
||||
kieSession.insert(nerEntities);
|
||||
|
||||
kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
|
||||
kieSession.fireAllRules();
|
||||
|
||||
@ -24,7 +24,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.ImageServiceResponseAdapter;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
|
||||
@ -39,6 +38,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
|
||||
@ -159,8 +159,8 @@ public class AnalyzeService {
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
|
||||
}
|
||||
|
||||
List<EntityRecognitionEntity> nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
|
||||
log.info("Found {} Entity Recognition Service Entities", nerEntities.size());
|
||||
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
|
||||
log.info("Found {} Entity Recognition Service Entities", nerEntities.getNerEntityList().size());
|
||||
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
|
||||
log.info("Rules updated");
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
@ -200,13 +200,14 @@ public class AnalyzeService {
|
||||
}
|
||||
|
||||
|
||||
private List<EntityRecognitionEntity> getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
|
||||
private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
|
||||
|
||||
List<EntityRecognitionEntity> nerEntities;
|
||||
NerEntities nerEntities;
|
||||
if (redactionServiceSettings.isNerServiceEnabled()) {
|
||||
nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph);
|
||||
nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()),
|
||||
documentGraph));
|
||||
} else {
|
||||
nerEntities = Collections.emptyList();
|
||||
nerEntities = new NerEntities(Collections.emptyList());
|
||||
}
|
||||
return nerEntities;
|
||||
}
|
||||
@ -218,7 +219,7 @@ public class AnalyzeService {
|
||||
long startTime = System.currentTimeMillis();
|
||||
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
|
||||
List<EntityRecognitionEntity> nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
|
||||
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
|
||||
|
||||
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
|
||||
|
||||
@ -10,7 +10,7 @@ import org.springframework.stereotype.Component;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
@ -42,7 +42,7 @@ class EntityFinder {
|
||||
int sectionNumber,
|
||||
Dictionary dictionary,
|
||||
boolean local,
|
||||
NerEntities nerEntities,
|
||||
NerEntitiesModel nerEntitiesModel,
|
||||
List<Integer> cellStarts,
|
||||
ManualRedactions manualRedactions) {
|
||||
|
||||
@ -73,7 +73,7 @@ class EntityFinder {
|
||||
|
||||
Set<Entity> nerFound = new HashSet<>();
|
||||
if (!local) {
|
||||
nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
|
||||
nerFound.addAll(getNerValues(sectionNumber, nerEntitiesModel, cellStarts, headline));
|
||||
}
|
||||
|
||||
var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
|
||||
@ -81,12 +81,12 @@ class EntityFinder {
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
|
||||
private Set<Entity> getNerValues(int sectionNumber, NerEntitiesModel nerEntitiesModel, List<Integer> cellStarts, String headline) {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
|
||||
if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
|
||||
nerEntities.getData().get(sectionNumber).forEach(res -> {
|
||||
if (redactionServiceSettings.isNerServiceEnabled() && nerEntitiesModel.getData().containsKey(sectionNumber)) {
|
||||
nerEntitiesModel.getData().get(sectionNumber).forEach(res -> {
|
||||
if (cellStarts == null || cellStarts.isEmpty()) {
|
||||
entities.add(new Entity(res.getValue(),
|
||||
res.getType(),
|
||||
|
||||
@ -10,12 +10,12 @@ import org.springframework.stereotype.Service;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
|
||||
@ -40,7 +40,7 @@ public class EntityRedactionService {
|
||||
DocumentGraph documentGraph,
|
||||
KieContainer kieContainer,
|
||||
AnalyzeRequest analyzeRequest,
|
||||
List<EntityRecognitionEntity> nerEntities) {
|
||||
NerEntities nerEntities) {
|
||||
|
||||
log.debug("Starting Drools Execution");
|
||||
long droolsStart = System.currentTimeMillis();
|
||||
@ -60,7 +60,7 @@ public class EntityRedactionService {
|
||||
List<SemanticNode> sectionsToReanalyze,
|
||||
KieContainer kieContainer,
|
||||
AnalyzeRequest analyzeRequest,
|
||||
List<EntityRecognitionEntity> nerEntities) {
|
||||
NerEntities nerEntities) {
|
||||
|
||||
log.debug("Starting Drools execution");
|
||||
long ruleExecutionStart = System.currentTimeMillis();
|
||||
|
||||
@ -8,7 +8,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.Text;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
|
||||
@ -104,10 +104,10 @@ public class RedactionStorageService {
|
||||
|
||||
|
||||
@Timed("redactmanager_getNerEntities")
|
||||
public NerEntities getNerEntities(String dossierId, String fileId) {
|
||||
public NerEntitiesModel getNerEntities(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
|
||||
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntitiesModel.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
throw new NotFoundException("NER Entities are not available.");
|
||||
}
|
||||
|
||||
@ -41,6 +41,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
|
||||
@ -153,7 +154,7 @@ public class DocumentGraphPerformanceIntegrationTest extends BuildDocumentGraphI
|
||||
dictionary,
|
||||
Collections.emptyList(),
|
||||
new ManualRedactions(),
|
||||
Collections.emptyList());
|
||||
new NerEntities());
|
||||
System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart);
|
||||
|
||||
System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart);
|
||||
|
||||
@ -17,10 +17,8 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentGraphIntegrationTest;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
|
||||
@ -39,10 +37,10 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private static NerEntities parseNerEntities(String filePath) {
|
||||
private static NerEntitiesModel parseNerEntities(String filePath) {
|
||||
|
||||
ClassPathResource resource = new ClassPathResource(filePath);
|
||||
return ObjectMapperFactory.create().readValue(resource.getInputStream(), NerEntities.class);
|
||||
return ObjectMapperFactory.create().readValue(resource.getInputStream(), NerEntitiesModel.class);
|
||||
}
|
||||
|
||||
|
||||
@ -53,16 +51,17 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
|
||||
String filePath = "files/new/crafted document.pdf";
|
||||
String nerEntitiesFilePath = "ner_entities/crafted document.NER_ENTITIES.json";
|
||||
DocumentGraph documentGraph = buildGraphNoImages(filePath);
|
||||
List<EntityRecognitionEntity> entityRecognitionEntities = nerEntitiesAdapter.validateAndCombine(parseNerEntities(nerEntitiesFilePath), documentGraph);
|
||||
List<NerEntities.NerEntity> entityRecognitionEntities = validateAndCombine(parseNerEntities(nerEntitiesFilePath), documentGraph);
|
||||
assertFalse(entityRecognitionEntities.isEmpty());
|
||||
assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.getStartOffset() < entity.getEndOffset()));
|
||||
assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.boundary().start() < entity.boundary().end()));
|
||||
|
||||
ClassPathResource resource = new ClassPathResource(filePath);
|
||||
PDDocument pdDocument = PDDocument.load(resource.getInputStream());
|
||||
Stream<EntityRecognitionEntity> unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
|
||||
.filter(e -> !e.getType().equals("CBI_author"));
|
||||
Stream<NerEntities.NerEntity> unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
|
||||
.stream()
|
||||
.filter(e -> !e.type().equals("CBI_author"));
|
||||
List<RedactionEntity> redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts)
|
||||
.map(e -> entityCreationService.byBoundary(new Boundary(e.getStartOffset(), e.getEndOffset()), e.getType(), EntityType.ENTITY, documentGraph))
|
||||
.map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, documentGraph))
|
||||
.toList();
|
||||
redactionEntities.stream()
|
||||
.collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber()))
|
||||
@ -79,6 +78,18 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
private List<NerEntities.NerEntity> validateAndCombine(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
|
||||
|
||||
NerEntities nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(nerEntitiesModel, documentGraph));
|
||||
|
||||
List<NerEntities.NerEntity> cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList();
|
||||
Stream<NerEntities.NerEntity> cbiAddress = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
|
||||
.map(boundary -> new NerEntities.NerEntity(documentGraph.buildTextBlock().subSequence(boundary).toString(), boundary, "CBI_address"));
|
||||
|
||||
return Stream.concat(cbiAuthors.stream(), cbiAddress).toList();
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle2D> getPositionsFromEntities(Stream<RedactionEntity> entities) {
|
||||
|
||||
return entities.map(RedactionEntity::getRedactionPositionsPerPage)
|
||||
|
||||
@ -10,15 +10,15 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
|
||||
import java.util.LinkedList;
|
||||
import java.util.HashSet;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import java.util.Set
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
|
||||
@ -28,20 +28,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
|
||||
import java.util.stream.Collectors
|
||||
import java.util.Collection
|
||||
import java.util.stream.Stream
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.Collection;
|
||||
import java.util.stream.Stream;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
|
||||
|
||||
global DocumentGraph document
|
||||
global EntityCreationService entityCreationService
|
||||
global ManualRedactionApplicationService manualRedactionApplicationService
|
||||
global NerEntitiesAdapter nerEntitiesAdapter
|
||||
global Dictionary dictionary
|
||||
|
||||
// --------------------------------------- queries -------------------------------------------------------------------
|
||||
@ -50,18 +51,6 @@ query "getFileAttributes"
|
||||
$fileAttribute: FileAttribute()
|
||||
end
|
||||
|
||||
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
|
||||
|
||||
rule "add NER Entities of type CBI_author or CBI_address"
|
||||
salience 999
|
||||
when
|
||||
$nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address"))
|
||||
then
|
||||
RedactionEntity redactionEntity = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document);
|
||||
redactionEntity.addEngine(Engine.NER);
|
||||
insert(redactionEntity);
|
||||
end
|
||||
|
||||
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
||||
|
||||
rule "0: Expand CBI_author entities with firstname initials"
|
||||
@ -526,6 +515,31 @@ rule "102: Guidelines FileAttributes"
|
||||
.forEach(fileAttribute -> insert(fileAttribute));
|
||||
end
|
||||
|
||||
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
|
||||
|
||||
rule "add NER Entities of type CBI_author"
|
||||
salience 999
|
||||
when
|
||||
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
|
||||
then
|
||||
nerEntities.streamEntitiesOfType("CBI_author")
|
||||
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
|
||||
.forEach(entity -> insert(entity));
|
||||
end
|
||||
|
||||
rule "combine and add NER Entities as CBI_address"
|
||||
salience 999
|
||||
when
|
||||
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
|
||||
then
|
||||
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
|
||||
.map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
|
||||
.forEach(entity -> {
|
||||
entity.addEngine(Engine.NER);
|
||||
insert(entity);
|
||||
});
|
||||
end
|
||||
|
||||
// --------------------------------------- manual redaction rules -------------------------------------------------------------------
|
||||
|
||||
rule "Apply manual resize redaction"
|
||||
@ -632,6 +646,7 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
|
||||
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
|
||||
$recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
||||
then
|
||||
$entity.addEngines($recommendation.getEngines());
|
||||
$recommendation.removeFromGraph();
|
||||
retract($recommendation);
|
||||
end
|
||||
|
||||
@ -10,15 +10,15 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
|
||||
import java.util.LinkedList;
|
||||
import java.util.HashSet;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import java.util.Set
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
|
||||
@ -28,20 +28,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
|
||||
import java.util.stream.Collectors
|
||||
import java.util.Collection
|
||||
import java.util.stream.Stream
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.Collection;
|
||||
import java.util.stream.Stream;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
|
||||
|
||||
global DocumentGraph document
|
||||
global EntityCreationService entityCreationService
|
||||
global ManualRedactionApplicationService manualRedactionApplicationService
|
||||
global NerEntitiesAdapter nerEntitiesAdapter
|
||||
global Dictionary dictionary
|
||||
|
||||
// --------------------------------------- queries -------------------------------------------------------------------
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user