RED-8702: Explore document databases to store entityLog
* wip - rewrite entity log creation and update with mongodb
This commit is contained in:
maverickstuder
parent
e26c492ba8
commit
9a836387be
@ -1,5 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.service;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService.getRelevantManuallyModifiedAnnotationIds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
@ -20,6 +23,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
|
||||
@ -80,7 +84,7 @@ public class AnalyzeService {
|
||||
public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
EntityLog entityLogWithoutEntries = redactionStorageService.getEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
@ -90,25 +94,37 @@ public class AnalyzeService {
|
||||
log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
// not yet ready for reanalysis
|
||||
if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
|
||||
if (entityLogWithoutEntries == null || document == null || document.getNumberOfPages() == 0) {
|
||||
return analyze(analyzeRequest);
|
||||
}
|
||||
|
||||
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
|
||||
new DictionaryVersion(previousEntityLog.getDictionaryVersion(),
|
||||
previousEntityLog.getDossierDictionaryVersion()),
|
||||
new DictionaryVersion(entityLogWithoutEntries.getDictionaryVersion(),
|
||||
entityLogWithoutEntries.getDossierDictionaryVersion()),
|
||||
analyzeRequest.getDossierId());
|
||||
|
||||
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement, importedRedactions);
|
||||
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
|
||||
|
||||
List<EntityLogEntry> relevantEntityLogEntries = redactionStorageService.findEntriesWithContainingNodeIds(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
relevantManuallyModifiedAnnotationIds);
|
||||
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest,
|
||||
document,
|
||||
dictionaryIncrement,
|
||||
importedRedactions,
|
||||
relevantEntityLogEntries,
|
||||
relevantManuallyModifiedAnnotationIds);
|
||||
|
||||
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
|
||||
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
if (sectionsToReAnalyse.isEmpty()) {
|
||||
|
||||
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
|
||||
// todo: 8702 - this feels wrong
|
||||
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
|
||||
dictionaryIncrement.getDictionaryVersion(),
|
||||
analyzeRequest,
|
||||
false);
|
||||
new ArrayList<>(), new ArrayList<>());
|
||||
|
||||
return finalizeAnalysis(analyzeRequest,
|
||||
startTime,
|
||||
@ -154,7 +170,7 @@ public class AnalyzeService {
|
||||
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
|
||||
document,
|
||||
notFoundManualOrImportedEntries,
|
||||
previousEntityLog,
|
||||
entityLogWithoutEntries,
|
||||
sectionsToReanalyseIds,
|
||||
dictionary.getVersion());
|
||||
|
||||
@ -228,7 +244,7 @@ public class AnalyzeService {
|
||||
return finalizeAnalysis(analyzeRequest,
|
||||
startTime,
|
||||
kieWrapperComponentRules,
|
||||
new EntityLogChanges(entityLog, false),
|
||||
EntityLogChanges.builder().entityLog(entityLog).build(),
|
||||
document,
|
||||
document.getNumberOfPages(),
|
||||
dictionary.getVersion(),
|
||||
@ -248,7 +264,7 @@ public class AnalyzeService {
|
||||
Set<FileAttribute> addedFileAttributes) {
|
||||
|
||||
EntityLog entityLog = entityLogChanges.getEntityLog();
|
||||
//redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
|
||||
|
||||
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog());
|
||||
|
||||
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
@ -317,12 +333,18 @@ public class AnalyzeService {
|
||||
|
||||
|
||||
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest,
|
||||
EntityLog entityLog,
|
||||
Document document,
|
||||
DictionaryIncrement dictionaryIncrement,
|
||||
ImportedRedactions importedRedactions) {
|
||||
ImportedRedactions importedRedactions,
|
||||
List<EntityLogEntry> entityLogEntries,
|
||||
Set<String> relevantManuallyModifiedAnnotationIds) {
|
||||
|
||||
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest, importedRedactions);
|
||||
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement,
|
||||
document,
|
||||
analyzeRequest,
|
||||
importedRedactions,
|
||||
entityLogEntries,
|
||||
relevantManuallyModifiedAnnotationIds);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -95,7 +95,7 @@ public class EntityLogCreatorService {
|
||||
}
|
||||
|
||||
|
||||
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, boolean hasChanges) {
|
||||
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, List<EntityLogEntry> newEntries, List<EntityLogEntry> updatedEntries) {
|
||||
|
||||
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
|
||||
entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
||||
@ -104,7 +104,7 @@ public class EntityLogCreatorService {
|
||||
entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
|
||||
entityLog.setAnalysisNumber(analyzeRequest.getAnalysisNumber());
|
||||
|
||||
return new EntityLogChanges(entityLog, hasChanges);
|
||||
return EntityLogChanges.builder().entityLog(entityLog).newEntityLogEntries(newEntries).updatedEntityLogEntries(updatedEntries).build();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -11,7 +11,6 @@ import java.util.stream.Stream;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
|
||||
@ -44,23 +43,17 @@ public class SectionFinderService {
|
||||
|
||||
@Timed("redactmanager_findSectionsToReanalyse")
|
||||
public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement,
|
||||
EntityLog entityLog,
|
||||
Document document,
|
||||
AnalyzeRequest analyzeRequest,
|
||||
ImportedRedactions importedRedactions) {
|
||||
ImportedRedactions importedRedactions,
|
||||
List<EntityLogEntry> entityLogEntries,
|
||||
Set<String> relevantManuallyModifiedAnnotationIds) {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
|
||||
Set<Integer> sectionsToReanalyse = new HashSet<>();
|
||||
for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
|
||||
if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
|
||||
if (entry.getContainingNodeId().isEmpty()) {
|
||||
continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
|
||||
}
|
||||
sectionsToReanalyse.add(entry.getContainingNodeId()
|
||||
.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
entityLogEntries.forEach(entityLogEntry -> sectionsToReanalyse.add(entityLogEntry.getContainingNodeId()
|
||||
.get(0)));
|
||||
|
||||
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
|
||||
.stream()
|
||||
@ -133,7 +126,7 @@ public class SectionFinderService {
|
||||
}
|
||||
|
||||
|
||||
private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
|
||||
public static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return new HashSet<>();
|
||||
|
||||
@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.storage;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@ -10,6 +11,7 @@ import org.springframework.cache.annotation.Cacheable;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
@ -142,8 +144,8 @@ public class RedactionStorageService {
|
||||
public EntityLog getEntityLog(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
//EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
|
||||
EntityLog entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId).orElseThrow(() -> new StorageObjectDoesNotExist(""));
|
||||
EntityLog entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId)
|
||||
.orElseThrow(() -> new StorageObjectDoesNotExist(""));
|
||||
entityLog.setEntityLogEntry(entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty()))
|
||||
@ -157,6 +159,26 @@ public class RedactionStorageService {
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getRedactionLog")
|
||||
public EntityLog getEntityLogWithoutEntries(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
return entityLogMongoService.findEntityLogWithoutEntries(dossierId, fileId)
|
||||
.orElseThrow(() -> new StorageObjectDoesNotExist(""));
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("EntityLog not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public List<EntityLogEntry> findEntriesWithContainingNodeIds(String dossierId, String fileId, Collection<String> entryIds) {
|
||||
|
||||
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
|
||||
}
|
||||
|
||||
|
||||
// !Warning! before activating redis cache you need to set
|
||||
// -Dio.netty.noPreferDirect=true -XX:MaxDirectMemorySize=1000M
|
||||
// Jvm args to the largest document data size we want to process. for 4443 pages file that was 500mb.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user