Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-9123: Improve performance of re-analysis (Spike)

Browse Source 
* refactoring and more tracing
This commit is contained in:
maverickstuder 2024-07-04 13:26:57 +02:00
parent 9abfb0d7f5
commit 03571697bf
5 changed files with 193 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java Normal file
View File

@ -0,0 +1,134 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.components.ComponentLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class AnalysisFinalizationService {
RedactionStorageService redactionStorageService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
ComponentLogCreatorService componentLogCreatorService;
RedactionServiceSettings redactionServiceSettings;
ComponentDroolsExecutionService componentDroolsExecutionService;
@Timed("redactmanager_finalizeAnalysis")
@Observed(name = "AnalysisFinalizationService", contextualName = "finalize-analysis")
public AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
KieWrapper kieWrapperComponentRules,
EntityLogChanges entityLogChanges,
Document document,
int numberOfPages,
boolean isReanalysis,
Set<FileAttribute> addedFileAttributes,
Context context) {
EntityLog entityLog = entityLogChanges.getEntityLog();
// as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
// also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
} else {
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
}
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
}
}
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
long duration = System.currentTimeMillis() - startTime;
redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
return AnalyzeResult.builder()
.dossierId(analyzeRequest.getDossierId())
.fileId(analyzeRequest.getFileId())
.duration(duration)
.numberOfPages(numberOfPages)
.hasUpdates(entityLogChanges.hasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(entityLog.getRulesVersion())
.componentRulesVersion(kieWrapperComponentRules.rulesVersion())
.dictionaryVersion(entityLog.getDictionaryVersion())
.legalBasisVersion(entityLog.getLegalBasisVersion())
.dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
.wasReanalyzed(isReanalysis)
.manualRedactions(analyzeRequest.getManualRedactions())
.addedFileAttributes(addedFileAttributes)
.usedComponentMappings(analyzeRequest.getComponentMappings())
.build();
}
private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
KieWrapper kieWrapperComponentRules,
Document document,
Set<FileAttribute> addedFileAttributes,
EntityLog entityLog,
Context context) {
if (!kieWrapperComponentRules.isPresent()) {
return;
}
// We need the latest EntityLog entries for components rules execution
entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
entityLog,
document,
addedFileAttributes,
analyzeRequest.getComponentMappings(),
context);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
}

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java
View File

@ -37,6 +37,8 @@ import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCrea
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@ -81,6 +83,8 @@ public class AnalysisPreparationService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_getAnalysisData")
@Observed(name = "AnalysisPreparationService", contextualName = "get-analysis-data")
public AnalysisData getAnalysisData(AnalyzeRequest analyzeRequest) { public AnalysisData getAnalysisData(AnalyzeRequest analyzeRequest) {
CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor); CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor);
@ -118,6 +122,8 @@ public class AnalysisPreparationService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_getReanalysisSetupData")
@Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-setup-data")
public ReanalysisSetupData getReanalysisSetupData(AnalyzeRequest analyzeRequest) { public ReanalysisSetupData getReanalysisSetupData(AnalyzeRequest analyzeRequest) {
CompletableFuture<EntityLog> entityLogFuture = CompletableFuture.supplyAsync(() -> getEntityLog(analyzeRequest), taskExecutor); CompletableFuture<EntityLog> entityLogFuture = CompletableFuture.supplyAsync(() -> getEntityLog(analyzeRequest), taskExecutor);
@ -131,6 +137,8 @@ public class AnalysisPreparationService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_getReanalysisInitialProcessingData")
@Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-initial-processing-data")
public ReanalysisInitialProcessingData getReanalysisInitialProcessingData(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData) { public ReanalysisInitialProcessingData getReanalysisInitialProcessingData(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData) {
CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor); CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor);
@ -157,6 +165,8 @@ public class AnalysisPreparationService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_getReanalysisFinalProcessingData")
@Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-final-processing-data")
public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest, public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest,
ReanalysisSetupData reanalysisSetupData, ReanalysisSetupData reanalysisSetupData,
ReanalysisInitialProcessingData reanalysisInitialProcessingData) { ReanalysisInitialProcessingData reanalysisInitialProcessingData) {
@ -181,7 +191,7 @@ public class AnalysisPreparationService {
} }
public KieWrapper getKieWrapper(AnalyzeRequest analyzeRequest, RuleFileType ruleFileType) { private KieWrapper getKieWrapper(AnalyzeRequest analyzeRequest, RuleFileType ruleFileType) {
KieWrapper kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), ruleFileType); KieWrapper kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), ruleFileType);
log.info("Updated {} Rules to Version {} for file {} in dossier {}", log.info("Updated {} Rules to Version {} for file {} in dossier {}",
@ -193,7 +203,7 @@ public class AnalysisPreparationService {
} }
public Document getDocument(AnalyzeRequest analyzeRequest) { private Document getDocument(AnalyzeRequest analyzeRequest) {
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -201,7 +211,7 @@ public class AnalysisPreparationService {
} }
public ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) { private ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) {
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -209,7 +219,7 @@ public class AnalysisPreparationService {
} }
public NerEntities getNerEntities(AnalyzeRequest analyzeRequest, Document document) { private NerEntities getNerEntities(AnalyzeRequest analyzeRequest, Document document) {
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -217,7 +227,7 @@ public class AnalysisPreparationService {
} }
public NerEntities getNerEntitiesFiltered(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) { private NerEntities getNerEntitiesFiltered(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) {
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

160
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -9,31 +9,19 @@ import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.AnalysisPreparationService.AnalysisData; import com.iqser.red.service.redaction.v1.server.service.AnalysisPreparationService.AnalysisData;
import com.iqser.red.service.redaction.v1.server.service.components.ComponentLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
@ -51,15 +39,11 @@ import lombok.extern.slf4j.Slf4j;
public class AnalyzeService { public class AnalyzeService {
EntityDroolsExecutionService entityDroolsExecutionService; EntityDroolsExecutionService entityDroolsExecutionService;
ComponentDroolsExecutionService componentDroolsExecutionService;
DictionarySearchService dictionarySearchService; DictionarySearchService dictionarySearchService;
EntityLogCreatorService entityLogCreatorService; EntityLogCreatorService entityLogCreatorService;
ComponentLogCreatorService componentLogCreatorService;
RedactionStorageService redactionStorageService;
RedactionServiceSettings redactionServiceSettings;
NotFoundImportedEntitiesService notFoundImportedEntitiesService; NotFoundImportedEntitiesService notFoundImportedEntitiesService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
AnalysisPreparationService analysisPreparationService; AnalysisPreparationService analysisPreparationService;
AnalysisFinalizationService analysisFinalizationService;
@Timed("redactmanager_reanalyze") @Timed("redactmanager_reanalyze")
@ -92,15 +76,15 @@ public class AnalyzeService {
new ArrayList<>(), new ArrayList<>(),
new ArrayList<>()); new ArrayList<>());
return finalizeAnalysis(analyzeRequest, return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
startTime, startTime,
initialProcessingData.kieWrapperComponentRules(), initialProcessingData.kieWrapperComponentRules(),
entityLogChanges, entityLogChanges,
setupData.document(), setupData.document(),
setupData.document().getNumberOfPages(), setupData.document().getNumberOfPages(),
true, true,
Collections.emptySet(), Collections.emptySet(),
context); context);
} }
context.setRuleVersion(initialProcessingData.kieWrapperEntityRules().rulesVersion()); context.setRuleVersion(initialProcessingData.kieWrapperEntityRules().rulesVersion());
@ -134,15 +118,15 @@ public class AnalyzeService {
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries());
return finalizeAnalysis(analyzeRequest, return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
startTime, startTime,
initialProcessingData.kieWrapperComponentRules(), initialProcessingData.kieWrapperComponentRules(),
entityLogChanges, entityLogChanges,
setupData.document(), setupData.document(),
setupData.document().getNumberOfPages(), setupData.document().getNumberOfPages(),
true, true,
new HashSet<>(allFileAttributes), new HashSet<>(allFileAttributes),
context); context);
} }
@ -187,103 +171,15 @@ public class AnalyzeService {
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());
return finalizeAnalysis(analyzeRequest, return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
startTime, startTime,
analysisData.kieWrapperComponentRules(), analysisData.kieWrapperComponentRules(),
entityLogChanges, entityLogChanges,
analysisData.document(), analysisData.document(),
analysisData.document().getNumberOfPages(), analysisData.document().getNumberOfPages(),
false, false,
new HashSet<>(allFileAttributes), new HashSet<>(allFileAttributes),
context); context);
}
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
KieWrapper kieWrapperComponentRules,
EntityLogChanges entityLogChanges,
Document document,
int numberOfPages,
boolean isReanalysis,
Set<FileAttribute> addedFileAttributes,
Context context) {
EntityLog entityLog = entityLogChanges.getEntityLog();
// as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
// also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
} else {
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
}
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
}
}
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
long duration = System.currentTimeMillis() - startTime;
redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
return AnalyzeResult.builder()
.dossierId(analyzeRequest.getDossierId())
.fileId(analyzeRequest.getFileId())
.duration(duration)
.numberOfPages(numberOfPages)
.hasUpdates(entityLogChanges.hasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(entityLog.getRulesVersion())
.componentRulesVersion(kieWrapperComponentRules.rulesVersion())
.dictionaryVersion(entityLog.getDictionaryVersion())
.legalBasisVersion(entityLog.getLegalBasisVersion())
.dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
.wasReanalyzed(isReanalysis)
.manualRedactions(analyzeRequest.getManualRedactions())
.addedFileAttributes(addedFileAttributes)
.usedComponentMappings(analyzeRequest.getComponentMappings())
.build();
}
private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
KieWrapper kieWrapperComponentRules,
Document document,
Set<FileAttribute> addedFileAttributes,
EntityLog entityLog,
Context context) {
if (!kieWrapperComponentRules.isPresent()) {
return;
}
// We need the latest EntityLog entries for components rules execution
entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
entityLog,
document,
addedFileAttributes,
analyzeRequest.getComponentMappings(),
context);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
} }
} }

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -43,6 +43,8 @@ import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils; import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils;
import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@ -70,6 +72,9 @@ public class EntityLogCreatorService {
} }
@Timed("redactmanager_createInitialEntityLog")
@Observed(name = "EntityLogCreatorService", contextualName = "create-initial-entity-log")
public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest, public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
List<PrecursorEntity> notFoundEntities, List<PrecursorEntity> notFoundEntities,
@ -109,6 +114,9 @@ public class EntityLogCreatorService {
} }
@Timed("redactmanager_updateVersionsAndReturnChanges")
@Observed(name = "EntityLogCreatorService", contextualName = "update-versions-and-return-changes")
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
DictionaryVersion dictionaryVersion, DictionaryVersion dictionaryVersion,
AnalyzeRequest analyzeRequest, AnalyzeRequest analyzeRequest,
@ -126,6 +134,8 @@ public class EntityLogCreatorService {
} }
@Timed("redactmanager_updatePreviousEntityLog")
@Observed(name = "EntityLogCreatorService", contextualName = "update-previous-entity-log")
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
EntityLog entityLogWithoutEntries, EntityLog entityLogWithoutEntries,

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java
View File

@ -9,8 +9,9 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -20,11 +21,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import org.springframework.stereotype.Service;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -37,6 +36,7 @@ public class NotFoundImportedEntitiesService {
@Timed("redactmanager_processEntityLog") @Timed("redactmanager_processEntityLog")
@Observed(name = "NotFoundImportedEntitiesService", contextualName = "process-entity-log")
public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) { public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
// recreate imported redactions with manual changes // recreate imported redactions with manual changes
if (notFoundEntities == null || notFoundEntities.isEmpty()) { if (notFoundEntities == null || notFoundEntities.isEmpty()) {
@ -65,7 +65,7 @@ public class NotFoundImportedEntitiesService {
} }
public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) { private Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>(); Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
Set<Integer> pageNumbers = importedEntities.stream() Set<Integer> pageNumbers = importedEntities.stream()
@ -82,7 +82,7 @@ public class NotFoundImportedEntitiesService {
} }
public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) { private void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
for (Position rectangle : entityLogEntry.getPositions()) { for (Position rectangle : entityLogEntry.getPositions()) {
if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) { if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {