Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* fix for all but one (some fixes are hotfixes though)
This commit is contained in:
maverickstuder 2024-03-27 14:47:10 +01:00
parent 9a836387be
commit 31def9fab3
4 changed files with 104 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -124,7 +124,8 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
dictionaryIncrement.getDictionaryVersion(),
analyzeRequest,
new ArrayList<>(), new ArrayList<>());
new ArrayList<>(),
new ArrayList<>());
return finalizeAnalysis(analyzeRequest,
startTime,
@ -169,8 +170,9 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
document,
notFoundManualOrImportedEntries,
entityLogWithoutEntries,
notFoundManualOrImportedEntries,
relevantEntityLogEntries,
sectionsToReanalyseIds,
dictionary.getVersion());
@ -233,18 +235,18 @@ public class AnalyzeService {
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
document,
notFoundManualOrImportedEntries,
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries);
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
return finalizeAnalysis(analyzeRequest,
startTime,
kieWrapperComponentRules,
EntityLogChanges.builder().entityLog(entityLog).build(),
entityLogChanges,
document,
document.getNumberOfPages(),
dictionary.getVersion(),
@ -265,10 +267,23 @@ public class AnalyzeService {
EntityLog entityLog = entityLogChanges.getEntityLog();
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog());
//hotfix for tests (todo 8702)
if (entityLog.getAnalysisNumber() <= 0 && !redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
redactionStorageService.insertEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
} else {
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
redactionStorageService.insertEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
}
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
}
}
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (entityLogChanges.isHasChanges() || !isReanalysis) {
if (entityLogChanges.hasChanges() || !isReanalysis) {
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion);
}
@ -283,7 +298,7 @@ public class AnalyzeService {
.fileId(analyzeRequest.getFileId())
.duration(duration)
.numberOfPages(numberOfPages)
.hasUpdates(entityLogChanges.isHasChanges())
.hasUpdates(entityLogChanges.hasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(entityLog.getRulesVersion())

45
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -32,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.AccessLevel;
@ -60,7 +61,7 @@ public class EntityLogCreatorService {
}
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<PrecursorEntity> notFoundEntities,
DictionaryVersion dictionaryVersion,
@ -72,16 +73,24 @@ public class EntityLogCreatorService {
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(),
previousExistingEntityLogEntries,
entityLogEntries,
analyzeRequest.getAnalysisNumber());
return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
return EntityLogChanges.builder()
.entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
dictionaryVersion.getDossierTemplateVersion(),
dictionaryVersion.getDossierVersion(),
rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())))
.updatedEntityLogEntries(entryChanges.updated())
.newEntityLogEntries(entryChanges.inserted())
.build();
}
@ -95,7 +104,11 @@ public class EntityLogCreatorService {
}
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, List<EntityLogEntry> newEntries, List<EntityLogEntry> updatedEntries) {
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
DictionaryVersion dictionaryVersion,
AnalyzeRequest analyzeRequest,
List<EntityLogEntry> newEntries,
List<EntityLogEntry> updatedEntries) {
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
@ -110,8 +123,9 @@ public class EntityLogCreatorService {
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document,
EntityLog entityLogWithoutEntries,
List<PrecursorEntity> notFoundEntries,
EntityLog previousEntityLog,
List<EntityLogEntry> relevantEntityLogEntries,
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
@ -123,18 +137,17 @@ public class EntityLogCreatorService {
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0))))
.collect(Collectors.toList());
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = new ArrayList<>(relevantEntityLogEntries);
previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newEntityIds));
previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithoutContainingNodeIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(),
previousEntriesFromReAnalyzedSections,
newEntityLogEntries,
analyzeRequest.getAnalysisNumber());
previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
}

37
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -87,6 +87,34 @@ public class RedactionStorageService {
}
@SneakyThrows
public void updateEntityLogWithoutEntries(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.saveEntityLogWithoutEntries(dossierId, fileId, entityLog);
}
@SneakyThrows
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.insertEntityLog(dossierId, fileId, entityLog);
}
@SneakyThrows
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
entityLogMongoService.insertEntityLogEntries(dossierId, fileId, entityLogEntries);
}
@SneakyThrows
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
entityLogMongoService.updateEntityLogEntries(dossierId, fileId, entityLogEntries);
}
@Timed("redactmanager_getImportedRedactions")
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
@ -177,6 +205,15 @@ public class RedactionStorageService {
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
}
public List<EntityLogEntry> findEntriesWithoutContainingNodeIds(String dossierId, String fileId) {
return entityLogMongoService.findAllEntityLogEntriesWithoutContainingNodeIds(dossierId, fileId);
}
public List<EntityLogEntry> findEntriesWithIds(String dossierId, String fileId, Collection<String> entryIds) {
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
}
// !Warning! before activating redis cache you need to set

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java
View File

@ -138,7 +138,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
document,
notFoundManualEntities,
new DictionaryVersion(),
0L).getEntityLogEntry();
0L).getEntityLog().getEntityLogEntry();
assertEquals(1, redactionLogEntries.size());
assertEquals(value, redactionLogEntries.get(0).getValue());