Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* fix for all but one (some fixes are hotfixes though)
This commit is contained in:
maverickstuder 2024-03-27 14:47:10 +01:00
parent 9a836387be
commit 31def9fab3
4 changed files with 104 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -124,7 +124,8 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries, EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
dictionaryIncrement.getDictionaryVersion(), dictionaryIncrement.getDictionaryVersion(),
analyzeRequest, analyzeRequest,
new ArrayList<>(), new ArrayList<>()); new ArrayList<>(),
new ArrayList<>());
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
@ -169,8 +170,9 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
document, document,
notFoundManualOrImportedEntries,
entityLogWithoutEntries, entityLogWithoutEntries,
notFoundManualOrImportedEntries,
relevantEntityLogEntries,
sectionsToReanalyseIds, sectionsToReanalyseIds,
dictionary.getVersion()); dictionary.getVersion());
@ -233,18 +235,18 @@ public class AnalyzeService {
nerEntities); nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest, EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
document, document,
notFoundManualOrImportedEntries, notFoundManualOrImportedEntries,
dictionary.getVersion(), dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion()); kieWrapperEntityRules.rulesVersion());
notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
kieWrapperComponentRules, kieWrapperComponentRules,
EntityLogChanges.builder().entityLog(entityLog).build(), entityLogChanges,
document, document,
document.getNumberOfPages(), document.getNumberOfPages(),
dictionary.getVersion(), dictionary.getVersion(),
@ -265,10 +267,23 @@ public class AnalyzeService {
EntityLog entityLog = entityLogChanges.getEntityLog(); EntityLog entityLog = entityLogChanges.getEntityLog();
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog()); //hotfix for tests (todo 8702)
if (entityLog.getAnalysisNumber() <= 0 && !redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
redactionStorageService.insertEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
} else {
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
redactionStorageService.insertEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
}
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
}
}
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (entityLogChanges.isHasChanges() || !isReanalysis) { if (entityLogChanges.hasChanges() || !isReanalysis) {
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion); computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion);
} }
@ -283,7 +298,7 @@ public class AnalyzeService {
.fileId(analyzeRequest.getFileId()) .fileId(analyzeRequest.getFileId())
.duration(duration) .duration(duration)
.numberOfPages(numberOfPages) .numberOfPages(numberOfPages)
.hasUpdates(entityLogChanges.isHasChanges()) .hasUpdates(entityLogChanges.hasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion()) .analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber()) .analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(entityLog.getRulesVersion()) .rulesVersion(entityLog.getRulesVersion())

45
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -32,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.AccessLevel; import lombok.AccessLevel;
@ -60,7 +61,7 @@ public class EntityLogCreatorService {
} }
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
List<PrecursorEntity> notFoundEntities, List<PrecursorEntity> notFoundEntities,
DictionaryVersion dictionaryVersion, DictionaryVersion dictionaryVersion,
@ -72,16 +73,24 @@ public class EntityLogCreatorService {
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber()); EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(),
previousExistingEntityLogEntries,
entityLogEntries,
analyzeRequest.getAnalysisNumber());
return new EntityLog(redactionServiceSettings.getAnalysisVersion(), return EntityLogChanges.builder()
.entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(), analyzeRequest.getAnalysisNumber(),
entityLogEntries, entityLogEntries,
toEntityLogLegalBasis(legalBasis), toEntityLogLegalBasis(legalBasis),
dictionaryVersion.getDossierTemplateVersion(), dictionaryVersion.getDossierTemplateVersion(),
dictionaryVersion.getDossierVersion(), dictionaryVersion.getDossierVersion(),
rulesVersion, rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())))
.updatedEntityLogEntries(entryChanges.updated())
.newEntityLogEntries(entryChanges.inserted())
.build();
} }
@ -95,7 +104,11 @@ public class EntityLogCreatorService {
} }
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, List<EntityLogEntry> newEntries, List<EntityLogEntry> updatedEntries) { public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
DictionaryVersion dictionaryVersion,
AnalyzeRequest analyzeRequest,
List<EntityLogEntry> newEntries,
List<EntityLogEntry> updatedEntries) {
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
@ -110,8 +123,9 @@ public class EntityLogCreatorService {
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
EntityLog entityLogWithoutEntries,
List<PrecursorEntity> notFoundEntries, List<PrecursorEntity> notFoundEntries,
EntityLog previousEntityLog, List<EntityLogEntry> relevantEntityLogEntries,
Set<Integer> sectionsToReanalyseIds, Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) { DictionaryVersion dictionaryVersion) {
@ -123,18 +137,17 @@ public class EntityLogCreatorService {
.map(EntityLogEntry::getId) .map(EntityLogEntry::getId)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() List<EntityLogEntry> previousEntriesFromReAnalyzedSections = new ArrayList<>(relevantEntityLogEntries);
.stream() previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newEntityIds));
.filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithoutContainingNodeIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
.get(0))))
.collect(Collectors.toList());
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(),
previousEntriesFromReAnalyzedSections,
newEntityLogEntries,
analyzeRequest.getAnalysisNumber());
previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
} }

37
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -87,6 +87,34 @@ public class RedactionStorageService {
} }
@SneakyThrows
public void updateEntityLogWithoutEntries(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.saveEntityLogWithoutEntries(dossierId, fileId, entityLog);
}
@SneakyThrows
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.insertEntityLog(dossierId, fileId, entityLog);
}
@SneakyThrows
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
entityLogMongoService.insertEntityLogEntries(dossierId, fileId, entityLogEntries);
}
@SneakyThrows
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
entityLogMongoService.updateEntityLogEntries(dossierId, fileId, entityLogEntries);
}
@Timed("redactmanager_getImportedRedactions") @Timed("redactmanager_getImportedRedactions")
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
@ -177,6 +205,15 @@ public class RedactionStorageService {
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds); return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
} }
public List<EntityLogEntry> findEntriesWithoutContainingNodeIds(String dossierId, String fileId) {
return entityLogMongoService.findAllEntityLogEntriesWithoutContainingNodeIds(dossierId, fileId);
}
public List<EntityLogEntry> findEntriesWithIds(String dossierId, String fileId, Collection<String> entryIds) {
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
}
// !Warning! before activating redis cache you need to set // !Warning! before activating redis cache you need to set

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java
View File

@ -138,7 +138,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
document, document,
notFoundManualEntities, notFoundManualEntities,
new DictionaryVersion(), new DictionaryVersion(),
0L).getEntityLogEntry(); 0L).getEntityLog().getEntityLogEntry();
assertEquals(1, redactionLogEntries.size()); assertEquals(1, redactionLogEntries.size());
assertEquals(value, redactionLogEntries.get(0).getValue()); assertEquals(value, redactionLogEntries.get(0).getValue());