Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-9123: Improve performance of re-analysis (Spike)

Browse Source 
* further async processing for finalize analysis as well as addDictionaryEntities
This commit is contained in:
maverickstuder 2024-07-08 13:58:19 +02:00
parent a4165608d4
commit e6a792377b
3 changed files with 91 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.springframework.stereotype.Service;
@ -56,6 +57,8 @@ public class AnalysisFinalizationService {
EntityLog entityLog = entityLogChanges.getEntityLog();
CompletableFuture.runAsync(() -> {
// as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
// also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
@ -74,6 +77,8 @@ public class AnalysisFinalizationService {
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
});
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
long duration = System.currentTimeMillis() - startTime;

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -39,14 +39,27 @@ public class DictionarySearchService {
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
for (DictionaryModel model : dictionary.getDictionaryModels()) {
bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
dictionary.getDictionaryModels()
.stream()
.parallel()
.forEach(model -> {
synchronized (node) {
bySearchImplementationAsDictionary(model.getEntriesSearch(),
model.getType(),
model.isHint() ? EntityType.HINT : EntityType.ENTITY,
node,
model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
model.getType(),
EntityType.FALSE_RECOMMENDATION,
node,
model.isDossierDictionary());
if (model.isDossierDictionary()) {
bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
}
}
});
}

69
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java
View File

@ -13,7 +13,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Path;
@ -33,7 +32,6 @@ import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
@ -45,6 +43,7 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.google.common.collect.Sets;
@ -58,7 +57,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@ -171,30 +169,22 @@ import lombok.extern.slf4j.Slf4j;
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
// analyzeService.analyze(analyzeRequest);
// EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
//
// // Serialize
// String jsonString = objectMapper.writeValueAsString(entityLog);
// // Serialize entityLog to a file
// try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser");
// ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
// out.writeObject(jsonString);
// } catch (IOException e) {
// e.printStackTrace();
// }
//
//analyzeService.analyze(analyzeRequest);
//storeEntityLogAsTempFile(analyzeRequest, objectMapper);
restoreEntityLogFromTempFile(objectMapper, analyzeRequest);
//Deserialize entityLog from a file
String entityLogString = null;
try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) {
entityLogString = (String) in.readObject();
} catch (IOException | ClassNotFoundException e) {
e.printStackTrace();
addManualRedactionEntryToFile(analyzeRequest);
analyzeService.reanalyze(analyzeRequest);
times.add(System.currentTimeMillis() - start);
}
var entityLog = objectMapper.readValue(entityLogString, EntityLog.class);
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
System.out.println("times in ms for each analyze run: " + times);
}
}
private static void addManualRedactionEntryToFile(AnalyzeRequest analyzeRequest) {
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId("yourAnnotationId");
@ -209,11 +199,34 @@ import lombok.extern.slf4j.Slf4j;
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build()));
analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
analyzeService.reanalyze(analyzeRequest);
times.add(System.currentTimeMillis() - start);
}
System.out.println("times in ms for each analyze run: " + times);
private void restoreEntityLogFromTempFile(ObjectMapper objectMapper, AnalyzeRequest analyzeRequest) throws JsonProcessingException {
//Deserialize entityLog from a file
String entityLogString = null;
try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) {
entityLogString = (String) in.readObject();
} catch (IOException | ClassNotFoundException e) {
e.printStackTrace();
}
var entityLog = objectMapper.readValue(entityLogString, EntityLog.class);
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
}
private void storeEntityLogAsTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException {
EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
// Serialize
String jsonString = objectMapper.writeValueAsString(entityLog);
// Serialize entityLog to a file
try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser");
ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
out.writeObject(jsonString);
} catch (IOException e) {
e.printStackTrace();
}
}