From e464864642a24f483610ffbbdc38b2081987c371 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Mon, 8 Jul 2024 17:06:55 +0200 Subject: [PATCH] RED-9123: Improve performance of re-analysis (Spike) * improved mongodb save/delete logic --- .../service/EntityLogCreatorService.java | 182 ------------------ 1 file changed, 182 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index f9afc85c..55062fc2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -212,25 +212,6 @@ public class EntityLogCreatorService { notFoundPrecursorEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>())))); - - - var otherEntries = new ArrayList<>(); - - document.getEntities() - .stream() - .filter(entity -> !entity.getValue().isEmpty()) - .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval) - .filter(entity -> !entity.removed()) - .forEach(entityNode -> otherEntries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); - - document.streamAllImages() - .filter(entity -> !entity.removed()) - .forEach(imageNode -> otherEntries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); - - notFoundPrecursorEntries.stream() - .filter(entity -> !entity.removed()) - .forEach(precursorEntity -> otherEntries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); - return entries; } @@ -492,167 +473,4 @@ public class EntityLogCreatorService { return manualChangesMap; } - - - - private List toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) { - - List entityLogEntries = new ArrayList<>(); - - // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities - for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { - - List rectanglesPerLine = positionOnPage.getRectanglePerLine() - .stream() - .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) - .toList(); - - EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId); - - // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages - entityLogEntry.setId(positionOnPage.getId()); - entityLogEntry.setPositions(rectanglesPerLine); - entityLogEntries.add(entityLogEntry); - } - - return entityLogEntries; - } - - - private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) { - - String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); - boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); - - List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId()); - List allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber); - - return EntityLogEntry.builder() - .id(image.getId()) - .value(image.getValue()) - .type(imageType) - .reason(image.buildReasonWithManualChangeDescriptions()) - .legalBasis(image.legalBasis()) - .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) - .dictionaryEntry(false) - .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) - .containingNodeId(image.getTreeId()) - .closestHeadline(image.getHeadline().getTextBlock().getSearchText()) - .section(image.getManualOverwrite().getSection() -// .orElse(image.getParent().toString())) - .orElse(this.buildSectionString(image.getParent()))) - .imageHasTransparency(image.isTransparent()) - .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) - .state(buildEntryState(image)) - .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE) - .engines(getEngines(null, image.getManualOverwrite())) - .paragraphPageIdx(-1) - .build(); - - } - - - private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) { - - String type = precursorEntity.getManualOverwrite().getType() - .orElse(precursorEntity.getType()); - - List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId()); - List allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); - - return EntityLogEntry.builder() - .id(precursorEntity.getId()) - .reason(precursorEntity.buildReasonWithManualChangeDescriptions()) - .legalBasis(precursorEntity.legalBasis()) - .value(precursorEntity.value()) - .type(type) - .state(buildEntryState(precursorEntity)) - .entryType(buildEntryType(precursorEntity)) - .section(precursorEntity.getManualOverwrite().getSection() - .orElse(precursorEntity.getSection())) - .containingNodeId(Collections.emptyList()) - .closestHeadline("") - .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString()) - .dictionaryEntry(precursorEntity.isDictionaryEntry()) - .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()) - .textAfter("") - .textBefore("") - .startOffset(-1) - .endOffset(-1) - .positions(precursorEntity.getManualOverwrite().getPositions() - .orElse(precursorEntity.getEntityPosition()) - .stream() - .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) - .toList()) - .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite())) - //imported is no longer used, frontend should check engines - //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) - .imported(false) - .reference(Collections.emptySet()) - .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) - .paragraphPageIdx(-1) - .build(); - } - - - private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) { - - Set referenceIds = new HashSet<>(); - entity.references() - .stream() - .filter(TextEntity::active) - .forEach(ref -> ref.getPositionsOnPagePerPage() - .forEach(pos -> referenceIds.add(pos.getId()))); - - EntryType entryType = buildEntryType(entity); - - List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id); - List allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); - - return EntityLogEntry.builder() - .reason(entity.buildReasonWithManualChangeDescriptions()) - .legalBasis(entity.legalBasis()) - .value(entity.getManualOverwrite().getValue() - .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) - .type(entity.type()) - .section(entity.getManualOverwrite().getSection() -// .orElse(entity.getDeepestFullyContainingNode().toString())) - .orElse(this.buildSectionString(entity.getDeepestFullyContainingNode()))) - .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId()) - .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) - .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) - .dictionaryEntry(entity.isDictionaryEntry()) - .textAfter(entity.getTextAfter()) - .textBefore(entity.getTextBefore()) - .startOffset(entity.getTextRange().start()) - .endOffset(entity.getTextRange().end()) - .duplicatedTextRanges(entity.getDuplicateTextRanges() - .stream() - .map(textRange -> DuplicatedTextRange.builder().start(textRange.start()).end(textRange.end()).build()) - .toList()) - .dossierDictionaryEntry(entity.isDossierDictionaryEntry()) - .engines(getEngines(entity.getEngines(), entity.getManualOverwrite())) - //imported is no longer used, frontend should check engines - //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED))) - .imported(false) - .reference(referenceIds) - .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) - .state(buildEntryState(entity)) - .entryType(entryType) - .paragraphPageIdx(determinePageParagraphIndex(entity, entryType)) - .build(); - } - - public List getManualChangesByEntityLogId(String dossierId, String fileId, String id) { - - List manualChanges = new ArrayList<>(); - List entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id)); - - for (EntityLogEntry entry : entityLogEntries) { - manualChanges.addAll(entry.getManualChanges()); - } - - return manualChanges; - } - }