Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009 - Document Tree Structure

Browse Source 
* removed ner entity validation due to poor performance
* filtered ner entities during reanalysis to improve performance
This commit is contained in:
Kilian Schuettler 2023-05-04 00:53:54 +02:00
parent e7d1fe242b
commit a9e3c1e467
6 changed files with 29967 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
View File

@ -18,6 +18,7 @@ import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
@ -180,7 +181,9 @@ public class EntityCreationService {
public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
return byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
RedactionEntity entity = byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
entity.addEngine(Engine.NER);
return entity;
}

22
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java
View File

@ -18,7 +18,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.te
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class NerEntitiesAdapter {
@ -33,20 +35,18 @@ public class NerEntitiesAdapter {
/**
* Adds the appropriate Offsets to the entities from the NER Service and validates, that the values match.
* Adds the appropriate Offsets to the entities from the NER Service.
*
* @param nerEntitiesModel the Entities just as the NER Service returns them
* @param documentGraph the document structure, from which the NER Service found the entities
* @return a stream of validated entities
*/
public List<NerEntities.NerEntity> getValidatedEntities(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntitiesModel) //
.filter(nerEntityModel -> nerEntityOffsetMatchesDocumentGraphOffsets(nerEntityModel, documentGraph.buildTextBlock()))
.map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
nerEntityModel.getType()))
.toList();
return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(
nerEntityModel.getValue(),
new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
nerEntityModel.getType())).toList());
}
@ -150,12 +150,6 @@ public class NerEntitiesAdapter {
}
private boolean nerEntityOffsetMatchesDocumentGraphOffsets(EntityRecognitionEntity nerEntity, TextBlock textBlock) {
return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset()));
}
private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) {
nerEntitiesModel.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> {

185
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
View File

@ -33,7 +33,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedSectionText;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedText;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
@ -135,92 +134,6 @@ public class AnalyzeService {
}
@Timed("redactmanager_reanalyze")
@SneakyThrows
public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
log.info("Start reanalysis for request: {}", analyzeRequest);
RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("loaded previous redaction log");
DocumentData documentData = redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("loaded document data");
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(documentData);
log.info("constructed graph");
// not yet ready for reanalysis
if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) {
return analyze(analyzeRequest);
}
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
analyzeRequest.getDossierId());
log.info("dictionary increment loaded {}", dictionaryIncrement);
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, redactionLog, documentGraph, dictionaryIncrement);
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(documentGraph, sectionsToReanalyseIds);
log.info("Should reanalyze {} sections for request: {}", sectionsToReAnalyse.size(), analyzeRequest);
if (sectionsToReAnalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
}
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
log.info("Found {} Entity Recognition Service Entities", nerEntities.getNerEntityList().size());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
log.info("Rules updated");
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Dictionary updated");
sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
log.info("Dictionary search finished");
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
log.info("Rule execution finished");
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newRedactionLogEntries,
false);
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, addedFileAttributes);
}
private static List<SemanticNode> getSectionsToReAnalyse(DocumentGraph documentGraph, Set<Integer> sectionsToReanalyseIds) {
return documentGraph.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))).collect(Collectors.toList());
}
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DocumentGraph documentGraph, DictionaryIncrement dictionaryIncrement) {
return analyzeRequest.getSectionsToReanalyse().isEmpty() //
? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
: analyzeRequest.getSectionsToReanalyse();
}
private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
NerEntities nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Get NerEntitiesModel");
nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(nerEntitiesModel, documentGraph));
} else {
nerEntities = new NerEntities(Collections.emptyList());
}
return nerEntities;
}
@Timed("redactmanager_analyze")
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
@ -263,6 +176,56 @@ public class AnalyzeService {
}
@Timed("redactmanager_reanalyze")
@SneakyThrows
public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
// not yet ready for reanalysis
if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) {
return analyze(analyzeRequest);
}
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
analyzeRequest.getDossierId());
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, redactionLog, documentGraph, dictionaryIncrement);
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(documentGraph, sectionsToReanalyseIds);
if (sectionsToReAnalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
}
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, documentGraph, sectionsToReanalyseIds);
log.info("Reanalyze {} sections with {} Ner Entities", sectionsToReAnalyse.size(), nerEntities.getNerEntityList().size());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
long ruleStart = System.currentTimeMillis();
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
log.info("Rule execution took {} ms", System.currentTimeMillis() - ruleStart);
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newRedactionLogEntries,
false);
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, addedFileAttributes);
}
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
RedactionLog redactionLog,
@ -305,6 +268,54 @@ public class AnalyzeService {
}
private static List<SemanticNode> getSectionsToReAnalyse(DocumentGraph documentGraph, Set<Integer> sectionsToReanalyseIds) {
return documentGraph.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))).collect(Collectors.toList());
}
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DocumentGraph documentGraph, DictionaryIncrement dictionaryIncrement) {
return analyzeRequest.getSectionsToReanalyse().isEmpty() //
? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
: analyzeRequest.getSectionsToReanalyse();
}
private NerEntities getEntityRecognitionEntitiesFilteredBySectionIds(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph, Set<Integer> sectionsToReanalyseIds) {
NerEntities nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
nerEntitiesModel = filterNerEntitiesModelBySectionIds(sectionsToReanalyseIds, nerEntitiesModel);
nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
} else {
nerEntities = new NerEntities(Collections.emptyList());
}
return nerEntities;
}
private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set<Integer> sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) {
return new NerEntitiesModel(nerEntitiesModel.getData().entrySet().stream() //
.filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
}
private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
NerEntities nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
nerEntities = nerEntitiesAdapter.toNerEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph);
} else {
nerEntities = new NerEntities(Collections.emptyList());
}
return nerEntities;
}
public List<RedactionLogLegalBasis> toSimplifiedSectionText(List<LegalBasis> legalBasis) {
return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());

50
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java
View File

@ -19,6 +19,7 @@ import org.springframework.core.io.ClassPathResource;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentGraphIntegrationTest;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
@ -27,7 +28,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
@Autowired
@ -57,7 +60,8 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
ClassPathResource resource = new ClassPathResource(filePath);
PDDocument pdDocument = PDDocument.load(resource.getInputStream());
Stream<NerEntities.NerEntity> unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
Stream<NerEntities.NerEntity> unchangedAddressParts = nerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
.getNerEntityList()
.stream()
.filter(e -> !e.type().equals("CBI_author"));
List<RedactionEntity> redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts)
@ -78,9 +82,51 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
}
@Test
@SneakyThrows
public void testGetNerEntitiesLargeFile() {
String filePath = "files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf";
String nerEntitiesFilePath = "ner_entities/S-Metolachlor_RAR_02_Volume_2_2018-09-06.NER_ENTITIES.json";
DocumentGraph documentGraph = buildGraphNoImages(filePath);
log.info("Graph built");
NerEntitiesModel nerEntitiesModel = parseNerEntities(nerEntitiesFilePath);
log.info("Parsed NerEntitiesModel");
NerEntities nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
log.info("Validated and mapped");
List<Boundary> nerEntityBoundaries = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).toList();
log.info("Combined to CBI_address");
List<RedactionEntity> cbiAddressEntities = nerEntityBoundaries.stream()
.map(b -> entityCreationService.byBoundary(b, "CBI_address", EntityType.RECOMMENDATION, documentGraph))
.toList();
assertFalse(cbiAddressEntities.isEmpty());
assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getBoundary().start() < entity.getBoundary().end()));
ClassPathResource resource = new ClassPathResource(filePath);
PDDocument pdDocument = PDDocument.load(resource.getInputStream());
List<RedactionEntity> validatedEntities = nerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
.getNerEntityList()
.stream()
.map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, documentGraph))
.toList();
Stream.concat(cbiAddressEntities.stream(), validatedEntities.stream())
.collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber()))
.forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber,
getPositionsFromEntityOfType("CBI_author", entities),
getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities),
getPositionsFromEntityOfType("CBI_address", entities),
pdDocument));
File outputFile = new File("/tmp/nerEntities.pdf");
pdDocument.save(outputFile);
pdDocument.close();
}
private List<NerEntities.NerEntity> validateAndCombine(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
NerEntities nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(nerEntitiesModel, documentGraph));
NerEntities nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
List<NerEntities.NerEntity> cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList();
Stream<NerEntities.NerEntity> cbiAddress = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -666,10 +666,10 @@ rule "remove Entity of lower rank, when equal boundaries and entityType"
rule "remove duplicate FileAttributes"
salience 64
when
$first: FileAttribute($label: label, $value: value)
$second: FileAttribute(this != $first, label == $label, value == $value)
$fileAttribute: FileAttribute($label: label, $value: value)
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
then
retract($second);
retract($duplicate);
end
// --------------------------------------- local dictionary search -------------------------------------------------------------------

29806
redaction-service-v1/redaction-service-server-v1/src/test/resources/ner_entities/S-Metolachlor_RAR_02_Volume_2_2018-09-06.NER_ENTITIES.json Normal file
View File

File diff suppressed because it is too large Load Diff