From 90b01144d90257643ad6153c5f1a0ef83241ddc4 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Mon, 8 Jul 2024 14:28:52 +0200 Subject: [PATCH] RED-9123: Improve performance of re-analysis (Spike) * fix typo and revert async finalize --- .../service/AnalysisFinalizationService.java | 29 ++- .../service/EntityLogCreatorService.java | 184 +++++++++++++++++- 2 files changed, 196 insertions(+), 17 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java index a7cabe8e..850975e9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java @@ -57,27 +57,24 @@ public class AnalysisFinalizationService { EntityLog entityLog = entityLogChanges.getEntityLog(); - CompletableFuture.runAsync(() -> { + // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used + // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed + if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) { + redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); - // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used - // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed - if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) { - redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + } else { + redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); - } else { - redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); - - if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) { - redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries()); - } - if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) { - redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries()); - } + if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) { + redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries()); } + if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) { + redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries()); + } + } - log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - }); computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 230591e1..f9afc85c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -210,7 +210,26 @@ public class EntityLogCreatorService { images.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, manualChangesMap.getOrDefault(imageNode.getId(), new ArrayList<>())))); - textEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>())))); + notFoundPrecursorEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>())))); + + + + var otherEntries = new ArrayList<>(); + + document.getEntities() + .stream() + .filter(entity -> !entity.getValue().isEmpty()) + .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval) + .filter(entity -> !entity.removed()) + .forEach(entityNode -> otherEntries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); + + document.streamAllImages() + .filter(entity -> !entity.removed()) + .forEach(imageNode -> otherEntries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); + + notFoundPrecursorEntries.stream() + .filter(entity -> !entity.removed()) + .forEach(precursorEntity -> otherEntries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); return entries; } @@ -473,4 +492,167 @@ public class EntityLogCreatorService { return manualChangesMap; } + + + + private List toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) { + + List entityLogEntries = new ArrayList<>(); + + // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities + for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { + + List rectanglesPerLine = positionOnPage.getRectanglePerLine() + .stream() + .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) + .toList(); + + EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId); + + // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages + entityLogEntry.setId(positionOnPage.getId()); + entityLogEntry.setPositions(rectanglesPerLine); + entityLogEntries.add(entityLogEntry); + } + + return entityLogEntries; + } + + + private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) { + + String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); + boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); + + List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId()); + List allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber); + + return EntityLogEntry.builder() + .id(image.getId()) + .value(image.getValue()) + .type(imageType) + .reason(image.buildReasonWithManualChangeDescriptions()) + .legalBasis(image.legalBasis()) + .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(false) + .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) + .containingNodeId(image.getTreeId()) + .closestHeadline(image.getHeadline().getTextBlock().getSearchText()) + .section(image.getManualOverwrite().getSection() +// .orElse(image.getParent().toString())) + .orElse(this.buildSectionString(image.getParent()))) + .imageHasTransparency(image.isTransparent()) + .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) + .state(buildEntryState(image)) + .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE) + .engines(getEngines(null, image.getManualOverwrite())) + .paragraphPageIdx(-1) + .build(); + + } + + + private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) { + + String type = precursorEntity.getManualOverwrite().getType() + .orElse(precursorEntity.getType()); + + List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId()); + List allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); + + return EntityLogEntry.builder() + .id(precursorEntity.getId()) + .reason(precursorEntity.buildReasonWithManualChangeDescriptions()) + .legalBasis(precursorEntity.legalBasis()) + .value(precursorEntity.value()) + .type(type) + .state(buildEntryState(precursorEntity)) + .entryType(buildEntryType(precursorEntity)) + .section(precursorEntity.getManualOverwrite().getSection() + .orElse(precursorEntity.getSection())) + .containingNodeId(Collections.emptyList()) + .closestHeadline("") + .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(precursorEntity.isDictionaryEntry()) + .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()) + .textAfter("") + .textBefore("") + .startOffset(-1) + .endOffset(-1) + .positions(precursorEntity.getManualOverwrite().getPositions() + .orElse(precursorEntity.getEntityPosition()) + .stream() + .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .toList()) + .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite())) + //imported is no longer used, frontend should check engines + //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) + .imported(false) + .reference(Collections.emptySet()) + .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) + .paragraphPageIdx(-1) + .build(); + } + + + private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) { + + Set referenceIds = new HashSet<>(); + entity.references() + .stream() + .filter(TextEntity::active) + .forEach(ref -> ref.getPositionsOnPagePerPage() + .forEach(pos -> referenceIds.add(pos.getId()))); + + EntryType entryType = buildEntryType(entity); + + List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id); + List allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); + + return EntityLogEntry.builder() + .reason(entity.buildReasonWithManualChangeDescriptions()) + .legalBasis(entity.legalBasis()) + .value(entity.getManualOverwrite().getValue() + .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) + .type(entity.type()) + .section(entity.getManualOverwrite().getSection() +// .orElse(entity.getDeepestFullyContainingNode().toString())) + .orElse(this.buildSectionString(entity.getDeepestFullyContainingNode()))) + .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId()) + .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) + .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(entity.isDictionaryEntry()) + .textAfter(entity.getTextAfter()) + .textBefore(entity.getTextBefore()) + .startOffset(entity.getTextRange().start()) + .endOffset(entity.getTextRange().end()) + .duplicatedTextRanges(entity.getDuplicateTextRanges() + .stream() + .map(textRange -> DuplicatedTextRange.builder().start(textRange.start()).end(textRange.end()).build()) + .toList()) + .dossierDictionaryEntry(entity.isDossierDictionaryEntry()) + .engines(getEngines(entity.getEngines(), entity.getManualOverwrite())) + //imported is no longer used, frontend should check engines + //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED))) + .imported(false) + .reference(referenceIds) + .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) + .state(buildEntryState(entity)) + .entryType(entryType) + .paragraphPageIdx(determinePageParagraphIndex(entity, entryType)) + .build(); + } + + public List getManualChangesByEntityLogId(String dossierId, String fileId, String id) { + + List manualChanges = new ArrayList<>(); + List entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id)); + + for (EntityLogEntry entry : entityLogEntries) { + manualChanges.addAll(entry.getManualChanges()); + } + + return manualChanges; + } + }