RED-9123: Improve performance of re-analysis (Spike)
* fix typo and revert async finalize
This commit is contained in:
maverickstuder
parent
e6a792377b
commit
90b01144d9
@ -57,27 +57,24 @@ public class AnalysisFinalizationService {
|
|||||||
|
|
||||||
EntityLog entityLog = entityLogChanges.getEntityLog();
|
EntityLog entityLog = entityLogChanges.getEntityLog();
|
||||||
|
|
||||||
CompletableFuture.runAsync(() -> {
|
// as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
|
||||||
|
// also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
|
||||||
|
if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
|
||||||
|
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
|
||||||
|
|
||||||
// as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
|
} else {
|
||||||
// also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
|
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
|
||||||
if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
|
|
||||||
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
|
|
||||||
|
|
||||||
} else {
|
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
|
||||||
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
|
redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
|
||||||
|
|
||||||
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
|
|
||||||
redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
|
|
||||||
}
|
|
||||||
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
|
|
||||||
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
|
||||||
|
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||||
|
|
||||||
});
|
|
||||||
|
|
||||||
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
|
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
|
||||||
|
|
||||||
|
|||||||
@ -210,7 +210,26 @@ public class EntityLogCreatorService {
|
|||||||
|
|
||||||
images.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, manualChangesMap.getOrDefault(imageNode.getId(), new ArrayList<>()))));
|
images.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, manualChangesMap.getOrDefault(imageNode.getId(), new ArrayList<>()))));
|
||||||
|
|
||||||
textEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>()))));
|
notFoundPrecursorEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>()))));
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
var otherEntries = new ArrayList<>();
|
||||||
|
|
||||||
|
document.getEntities()
|
||||||
|
.stream()
|
||||||
|
.filter(entity -> !entity.getValue().isEmpty())
|
||||||
|
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
|
||||||
|
.filter(entity -> !entity.removed())
|
||||||
|
.forEach(entityNode -> otherEntries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
|
||||||
|
|
||||||
|
document.streamAllImages()
|
||||||
|
.filter(entity -> !entity.removed())
|
||||||
|
.forEach(imageNode -> otherEntries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
|
||||||
|
|
||||||
|
notFoundPrecursorEntries.stream()
|
||||||
|
.filter(entity -> !entity.removed())
|
||||||
|
.forEach(precursorEntity -> otherEntries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
|
||||||
|
|
||||||
return entries;
|
return entries;
|
||||||
}
|
}
|
||||||
@ -473,4 +492,167 @@ public class EntityLogCreatorService {
|
|||||||
return manualChangesMap;
|
return manualChangesMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) {
|
||||||
|
|
||||||
|
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
|
||||||
|
|
||||||
|
// split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
|
||||||
|
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
|
||||||
|
|
||||||
|
List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
|
||||||
|
.stream()
|
||||||
|
.map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId);
|
||||||
|
|
||||||
|
// set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
|
||||||
|
entityLogEntry.setId(positionOnPage.getId());
|
||||||
|
entityLogEntry.setPositions(rectanglesPerLine);
|
||||||
|
entityLogEntries.add(entityLogEntry);
|
||||||
|
}
|
||||||
|
|
||||||
|
return entityLogEntries;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) {
|
||||||
|
|
||||||
|
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
|
||||||
|
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
|
||||||
|
|
||||||
|
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId());
|
||||||
|
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
|
||||||
|
|
||||||
|
return EntityLogEntry.builder()
|
||||||
|
.id(image.getId())
|
||||||
|
.value(image.getValue())
|
||||||
|
.type(imageType)
|
||||||
|
.reason(image.buildReasonWithManualChangeDescriptions())
|
||||||
|
.legalBasis(image.legalBasis())
|
||||||
|
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
|
||||||
|
.dictionaryEntry(false)
|
||||||
|
.positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
|
||||||
|
.containingNodeId(image.getTreeId())
|
||||||
|
.closestHeadline(image.getHeadline().getTextBlock().getSearchText())
|
||||||
|
.section(image.getManualOverwrite().getSection()
|
||||||
|
// .orElse(image.getParent().toString()))
|
||||||
|
.orElse(this.buildSectionString(image.getParent())))
|
||||||
|
.imageHasTransparency(image.isTransparent())
|
||||||
|
.manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
|
||||||
|
.state(buildEntryState(image))
|
||||||
|
.entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
||||||
|
.engines(getEngines(null, image.getManualOverwrite()))
|
||||||
|
.paragraphPageIdx(-1)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) {
|
||||||
|
|
||||||
|
String type = precursorEntity.getManualOverwrite().getType()
|
||||||
|
.orElse(precursorEntity.getType());
|
||||||
|
|
||||||
|
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId());
|
||||||
|
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
|
||||||
|
|
||||||
|
return EntityLogEntry.builder()
|
||||||
|
.id(precursorEntity.getId())
|
||||||
|
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
|
||||||
|
.legalBasis(precursorEntity.legalBasis())
|
||||||
|
.value(precursorEntity.value())
|
||||||
|
.type(type)
|
||||||
|
.state(buildEntryState(precursorEntity))
|
||||||
|
.entryType(buildEntryType(precursorEntity))
|
||||||
|
.section(precursorEntity.getManualOverwrite().getSection()
|
||||||
|
.orElse(precursorEntity.getSection()))
|
||||||
|
.containingNodeId(Collections.emptyList())
|
||||||
|
.closestHeadline("")
|
||||||
|
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
|
||||||
|
.dictionaryEntry(precursorEntity.isDictionaryEntry())
|
||||||
|
.dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
|
||||||
|
.textAfter("")
|
||||||
|
.textBefore("")
|
||||||
|
.startOffset(-1)
|
||||||
|
.endOffset(-1)
|
||||||
|
.positions(precursorEntity.getManualOverwrite().getPositions()
|
||||||
|
.orElse(precursorEntity.getEntityPosition())
|
||||||
|
.stream()
|
||||||
|
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
||||||
|
.toList())
|
||||||
|
.engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite()))
|
||||||
|
//imported is no longer used, frontend should check engines
|
||||||
|
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
|
||||||
|
.imported(false)
|
||||||
|
.reference(Collections.emptySet())
|
||||||
|
.manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
|
||||||
|
.paragraphPageIdx(-1)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) {
|
||||||
|
|
||||||
|
Set<String> referenceIds = new HashSet<>();
|
||||||
|
entity.references()
|
||||||
|
.stream()
|
||||||
|
.filter(TextEntity::active)
|
||||||
|
.forEach(ref -> ref.getPositionsOnPagePerPage()
|
||||||
|
.forEach(pos -> referenceIds.add(pos.getId())));
|
||||||
|
|
||||||
|
EntryType entryType = buildEntryType(entity);
|
||||||
|
|
||||||
|
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id);
|
||||||
|
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
|
||||||
|
|
||||||
|
return EntityLogEntry.builder()
|
||||||
|
.reason(entity.buildReasonWithManualChangeDescriptions())
|
||||||
|
.legalBasis(entity.legalBasis())
|
||||||
|
.value(entity.getManualOverwrite().getValue()
|
||||||
|
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
||||||
|
.type(entity.type())
|
||||||
|
.section(entity.getManualOverwrite().getSection()
|
||||||
|
// .orElse(entity.getDeepestFullyContainingNode().toString()))
|
||||||
|
.orElse(this.buildSectionString(entity.getDeepestFullyContainingNode())))
|
||||||
|
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
|
||||||
|
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
|
||||||
|
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
|
||||||
|
.dictionaryEntry(entity.isDictionaryEntry())
|
||||||
|
.textAfter(entity.getTextAfter())
|
||||||
|
.textBefore(entity.getTextBefore())
|
||||||
|
.startOffset(entity.getTextRange().start())
|
||||||
|
.endOffset(entity.getTextRange().end())
|
||||||
|
.duplicatedTextRanges(entity.getDuplicateTextRanges()
|
||||||
|
.stream()
|
||||||
|
.map(textRange -> DuplicatedTextRange.builder().start(textRange.start()).end(textRange.end()).build())
|
||||||
|
.toList())
|
||||||
|
.dossierDictionaryEntry(entity.isDossierDictionaryEntry())
|
||||||
|
.engines(getEngines(entity.getEngines(), entity.getManualOverwrite()))
|
||||||
|
//imported is no longer used, frontend should check engines
|
||||||
|
//(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
|
||||||
|
.imported(false)
|
||||||
|
.reference(referenceIds)
|
||||||
|
.manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
|
||||||
|
.state(buildEntryState(entity))
|
||||||
|
.entryType(entryType)
|
||||||
|
.paragraphPageIdx(determinePageParagraphIndex(entity, entryType))
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<ManualChange> getManualChangesByEntityLogId(String dossierId, String fileId, String id) {
|
||||||
|
|
||||||
|
List<ManualChange> manualChanges = new ArrayList<>();
|
||||||
|
List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id));
|
||||||
|
|
||||||
|
for (EntityLogEntry entry : entityLogEntries) {
|
||||||
|
manualChanges.addAll(entry.getManualChanges());
|
||||||
|
}
|
||||||
|
|
||||||
|
return manualChanges;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user