Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* fix for test failing due to bson npe (wrong projection)
This commit is contained in:
maverickstuder 2024-03-27 17:34:54 +01:00
parent a57cdf7157
commit dade37bf42
4 changed files with 17 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -105,22 +105,20 @@ public class AnalyzeService {
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
List<EntityLogEntry> relevantEntityLogEntries = redactionStorageService.findEntriesWithContainingNodeIds(analyzeRequest.getDossierId(), Set<Integer> sectionsToReanalyseIds = redactionStorageService.findIdsOfSectionsToReanalyse(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
relevantManuallyModifiedAnnotationIds); relevantManuallyModifiedAnnotationIds);
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, sectionsToReanalyseIds.addAll(getSectionsToReanalyseIds(analyzeRequest,
document, document,
dictionaryIncrement, dictionaryIncrement,
importedRedactions, importedRedactions,
relevantEntityLogEntries, relevantManuallyModifiedAnnotationIds));
relevantManuallyModifiedAnnotationIds);
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (sectionsToReAnalyse.isEmpty()) { if (sectionsToReAnalyse.isEmpty()) {
// todo: 8702 - this feels wrong
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries, EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
dictionaryIncrement.getDictionaryVersion(), dictionaryIncrement.getDictionaryVersion(),
analyzeRequest, analyzeRequest,
@ -172,7 +170,6 @@ public class AnalyzeService {
document, document,
entityLogWithoutEntries, entityLogWithoutEntries,
notFoundManualOrImportedEntries, notFoundManualOrImportedEntries,
relevantEntityLogEntries,
sectionsToReanalyseIds, sectionsToReanalyseIds,
dictionary.getVersion()); dictionary.getVersion());
@ -351,14 +348,12 @@ public class AnalyzeService {
Document document, Document document,
DictionaryIncrement dictionaryIncrement, DictionaryIncrement dictionaryIncrement,
ImportedRedactions importedRedactions, ImportedRedactions importedRedactions,
List<EntityLogEntry> entityLogEntries,
Set<String> relevantManuallyModifiedAnnotationIds) { Set<String> relevantManuallyModifiedAnnotationIds) {
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement,
document, document,
analyzeRequest, analyzeRequest,
importedRedactions, importedRedactions,
entityLogEntries,
relevantManuallyModifiedAnnotationIds); relevantManuallyModifiedAnnotationIds);
} }

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -75,9 +75,7 @@ public class EntityLogCreatorService {
EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(), EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(), //analyzeRequest.getFileId(),
previousExistingEntityLogEntries, previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
entityLogEntries,
analyzeRequest.getAnalysisNumber());
return EntityLogChanges.builder() return EntityLogChanges.builder()
.entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(), .entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(),
@ -125,7 +123,6 @@ public class EntityLogCreatorService {
Document document, Document document,
EntityLog entityLogWithoutEntries, EntityLog entityLogWithoutEntries,
List<PrecursorEntity> notFoundEntries, List<PrecursorEntity> notFoundEntries,
List<EntityLogEntry> relevantEntityLogEntries,
Set<Integer> sectionsToReanalyseIds, Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) { DictionaryVersion dictionaryVersion) {
@ -133,19 +130,14 @@ public class EntityLogCreatorService {
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0))) .get(0)))
.collect(Collectors.toList()); .collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = new ArrayList<>(relevantEntityLogEntries); List<EntityLogEntry> previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(),
previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newEntityIds)); analyzeRequest.getFileId(),
previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithoutContainingNodeIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); sectionsToReanalyseIds);
EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(), EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(), //analyzeRequest.getFileId(),
previousEntriesFromReAnalyzedSections, previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
newEntityLogEntries,
analyzeRequest.getAnalysisNumber());
return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated()); return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
} }

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java
View File

@ -46,14 +46,11 @@ public class SectionFinderService {
Document document, Document document,
AnalyzeRequest analyzeRequest, AnalyzeRequest analyzeRequest,
ImportedRedactions importedRedactions, ImportedRedactions importedRedactions,
List<EntityLogEntry> entityLogEntries,
Set<String> relevantManuallyModifiedAnnotationIds) { Set<String> relevantManuallyModifiedAnnotationIds) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
Set<Integer> sectionsToReanalyse = new HashSet<>(); Set<Integer> sectionsToReanalyse = new HashSet<>();
entityLogEntries.forEach(entityLogEntry -> sectionsToReanalyse.add(entityLogEntry.getContainingNodeId()
.get(0)));
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues() var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
.stream() .stream()

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -5,6 +5,7 @@ import java.io.FileInputStream;
import java.io.InputStream; import java.io.InputStream;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.springframework.cache.annotation.Cacheable; import org.springframework.cache.annotation.Cacheable;
@ -201,21 +202,19 @@ public class RedactionStorageService {
} }
public List<EntityLogEntry> findEntriesWithContainingNodeIds(String dossierId, String fileId, Collection<String> entryIds) { public Set<Integer> findIdsOfSectionsToReanalyse(String dossierId, String fileId, Collection<String> entryIds) {
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds); return entityLogMongoService.findFirstContainingNodeIdForEachEntry(dossierId, fileId, entryIds);
}
public List<EntityLogEntry> findEntriesWithoutContainingNodeIds(String dossierId, String fileId) {
return entityLogMongoService.findAllEntityLogEntriesWithoutContainingNodeIds(dossierId, fileId);
} }
public List<EntityLogEntry> findEntriesWithIds(String dossierId, String fileId, Collection<String> entryIds) {
return entityLogMongoService.findAllEntityLogEntriesWithEntryIdsIn(dossierId, fileId, entryIds); public List<EntityLogEntry> findEntriesContainedBySectionsOrNotContained(String dossierId, String fileId, Collection<Integer> sectionIds) {
return entityLogMongoService.findEntityLogEntriesNotContainedOrFirstContainedByElementInList(dossierId, fileId, sectionIds);
} }
// !Warning! before activating redis cache you need to set // !Warning! before activating redis cache you need to set
// -Dio.netty.noPreferDirect=true -XX:MaxDirectMemorySize=1000M // -Dio.netty.noPreferDirect=true -XX:MaxDirectMemorySize=1000M
// Jvm args to the largest document data size we want to process. for 4443 pages file that was 500mb. // Jvm args to the largest document data size we want to process. for 4443 pages file that was 500mb.