Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* fix for test failing due to bson npe (wrong projection)
This commit is contained in:
maverickstuder 2024-03-27 17:34:54 +01:00
parent a57cdf7157
commit dade37bf42
4 changed files with 17 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -105,22 +105,20 @@ public class AnalyzeService {
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
List<EntityLogEntry> relevantEntityLogEntries = redactionStorageService.findEntriesWithContainingNodeIds(analyzeRequest.getDossierId(),
Set<Integer> sectionsToReanalyseIds = redactionStorageService.findIdsOfSectionsToReanalyse(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
relevantManuallyModifiedAnnotationIds);
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest,
sectionsToReanalyseIds.addAll(getSectionsToReanalyseIds(analyzeRequest,
document,
dictionaryIncrement,
importedRedactions,
relevantEntityLogEntries,
relevantManuallyModifiedAnnotationIds);
relevantManuallyModifiedAnnotationIds));
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (sectionsToReAnalyse.isEmpty()) {
// todo: 8702 - this feels wrong
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
dictionaryIncrement.getDictionaryVersion(),
analyzeRequest,
@ -172,7 +170,6 @@ public class AnalyzeService {
document,
entityLogWithoutEntries,
notFoundManualOrImportedEntries,
relevantEntityLogEntries,
sectionsToReanalyseIds,
dictionary.getVersion());
@ -351,14 +348,12 @@ public class AnalyzeService {
Document document,
DictionaryIncrement dictionaryIncrement,
ImportedRedactions importedRedactions,
List<EntityLogEntry> entityLogEntries,
Set<String> relevantManuallyModifiedAnnotationIds) {
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement,
document,
analyzeRequest,
importedRedactions,
entityLogEntries,
relevantManuallyModifiedAnnotationIds);
}

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -75,9 +75,7 @@ public class EntityLogCreatorService {
EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(),
previousExistingEntityLogEntries,
entityLogEntries,
analyzeRequest.getAnalysisNumber());
previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
return EntityLogChanges.builder()
.entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(),
@ -125,7 +123,6 @@ public class EntityLogCreatorService {
Document document,
EntityLog entityLogWithoutEntries,
List<PrecursorEntity> notFoundEntries,
List<EntityLogEntry> relevantEntityLogEntries,
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
@ -133,19 +130,14 @@ public class EntityLogCreatorService {
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0)))
.collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = new ArrayList<>(relevantEntityLogEntries);
previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newEntityIds));
previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithoutContainingNodeIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
sectionsToReanalyseIds);
EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(),
previousEntriesFromReAnalyzedSections,
newEntityLogEntries,
analyzeRequest.getAnalysisNumber());
previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java
View File

@ -46,14 +46,11 @@ public class SectionFinderService {
Document document,
AnalyzeRequest analyzeRequest,
ImportedRedactions importedRedactions,
List<EntityLogEntry> entityLogEntries,
Set<String> relevantManuallyModifiedAnnotationIds) {
long start = System.currentTimeMillis();
Set<Integer> sectionsToReanalyse = new HashSet<>();
entityLogEntries.forEach(entityLogEntry -> sectionsToReanalyse.add(entityLogEntry.getContainingNodeId()
.get(0)));
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
.stream()

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -5,6 +5,7 @@ import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.cache.annotation.Cacheable;
@ -201,21 +202,19 @@ public class RedactionStorageService {
}
public List<EntityLogEntry> findEntriesWithContainingNodeIds(String dossierId, String fileId, Collection<String> entryIds) {
public Set<Integer> findIdsOfSectionsToReanalyse(String dossierId, String fileId, Collection<String> entryIds) {
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
}
public List<EntityLogEntry> findEntriesWithoutContainingNodeIds(String dossierId, String fileId) {
return entityLogMongoService.findAllEntityLogEntriesWithoutContainingNodeIds(dossierId, fileId);
return entityLogMongoService.findFirstContainingNodeIdForEachEntry(dossierId, fileId, entryIds);
}
public List<EntityLogEntry> findEntriesWithIds(String dossierId, String fileId, Collection<String> entryIds) {
return entityLogMongoService.findAllEntityLogEntriesWithEntryIdsIn(dossierId, fileId, entryIds);
public List<EntityLogEntry> findEntriesContainedBySectionsOrNotContained(String dossierId, String fileId, Collection<Integer> sectionIds) {
return entityLogMongoService.findEntityLogEntriesNotContainedOrFirstContainedByElementInList(dossierId, fileId, sectionIds);
}
// !Warning! before activating redis cache you need to set
// -Dio.netty.noPreferDirect=true -XX:MaxDirectMemorySize=1000M
// Jvm args to the largest document data size we want to process. for 4443 pages file that was 500mb.