Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* fix for all but one (some fixes are hotfixes though)
This commit is contained in:
maverickstuder 2024-03-27 14:47:10 +01:00
parent c1458fa041
commit a4988002cd
5 changed files with 138 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -124,7 +124,8 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
dictionaryIncrement.getDictionaryVersion(),
analyzeRequest,
new ArrayList<>(), new ArrayList<>());
new ArrayList<>(),
new ArrayList<>());
return finalizeAnalysis(analyzeRequest,
startTime,
@ -169,8 +170,9 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
document,
notFoundManualOrImportedEntries,
entityLogWithoutEntries,
notFoundManualOrImportedEntries,
relevantEntityLogEntries,
sectionsToReanalyseIds,
dictionary.getVersion());
@ -233,18 +235,18 @@ public class AnalyzeService {
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
document,
notFoundManualOrImportedEntries,
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
document,
notFoundManualOrImportedEntries,
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries);
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
return finalizeAnalysis(analyzeRequest,
startTime,
kieWrapperComponentRules,
EntityLogChanges.builder().entityLog(entityLog).build(),
entityLogChanges,
document,
document.getNumberOfPages(),
dictionary.getVersion(),
@ -265,10 +267,23 @@ public class AnalyzeService {
EntityLog entityLog = entityLogChanges.getEntityLog();
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog());
//hotfix for tests (todo 8702)
if (entityLog.getAnalysisNumber() <= 0 && !redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
redactionStorageService.insertEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
} else {
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
redactionStorageService.insertEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
}
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
}
}
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (entityLogChanges.isHasChanges() || !isReanalysis) {
if (entityLogChanges.hasChanges() || !isReanalysis) {
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion);
}
@ -283,7 +298,7 @@ public class AnalyzeService {
.fileId(analyzeRequest.getFileId())
.duration(duration)
.numberOfPages(numberOfPages)
.hasUpdates(entityLogChanges.isHasChanges())
.hasUpdates(entityLogChanges.hasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(entityLog.getRulesVersion())

47
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -1,7 +1,7 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.Comparator;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@ -13,9 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel;
@ -29,41 +26,58 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class EntityChangeLogService {
//EntityLogMongoService entityLogMongoService;
@Timed("redactmanager_computeChanges")
public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
public EntryChanges computeChanges(//String dossierId,
//String fileId,
List<EntityLogEntry> previousEntityLogEntries,
List<EntityLogEntry> newEntityLogEntries,
int analysisNumber) {
var now = OffsetDateTime.now();
if (previousEntityLogEntries.isEmpty()) {
newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)));
return true;
//entityLogMongoService.insertEntityLogEntries(dossierId, fileId, newEntityLogEntries);
return new EntryChanges(newEntityLogEntries, new ArrayList<>());
}
boolean hasChanges = false;
List<EntityLogEntry> toInsert = new ArrayList<>();
List<EntityLogEntry> toUpdate = new ArrayList<>();
for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
.filter(entry -> entry.getId().equals(entityLogEntry.getId()))
.findAny();
if (optionalPreviousEntity.isEmpty()) {
hasChanges = true;
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
toInsert.add(entityLogEntry);
continue;
}
EntityLogEntry previousEntity = optionalPreviousEntity.get();
entityLogEntry.getChanges().addAll(previousEntity.getChanges());
if (!previousEntity.getState().equals(entityLogEntry.getState())) {
hasChanges = true;
ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
toUpdate.add(entityLogEntry);
}
}
addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now);
return hasChanges;
//entityLogMongoService.insertEntityLogEntries(dossierId, fileId, toInsert);
//entityLogMongoService.updateEntityLogEntries(dossierId, fileId, toUpdate);
toUpdate.addAll(addRemovedEntriesAsRemoved(//dossierId, fileId,
previousEntityLogEntries, newEntityLogEntries, analysisNumber, now));
return new EntryChanges(toInsert, toUpdate);
}
private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber, OffsetDateTime now) {
private List<EntityLogEntry> addRemovedEntriesAsRemoved(//String dossierId,
//String fileId,
List<EntityLogEntry> previousEntityLogEntries,
List<EntityLogEntry> newEntityLogEntries,
int analysisNumber,
OffsetDateTime now) {
Set<String> existingIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
@ -73,7 +87,9 @@ public class EntityChangeLogService {
.toList();
removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
//entityLogMongoService.updateEntityLogEntries(dossierId, fileId, removedEntries);
newEntityLogEntries.addAll(removedEntries);
return removedEntries;
}
@ -97,4 +113,9 @@ public class EntityChangeLogService {
return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED));
}
public record EntryChanges(List<EntityLogEntry> inserted, List<EntityLogEntry> updated) {
}
}

65
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -32,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.AccessLevel;
@ -58,11 +59,11 @@ public class EntityLogCreatorService {
}
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<PrecursorEntity> notFoundEntities,
DictionaryVersion dictionaryVersion,
long rulesVersion) {
public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<PrecursorEntity> notFoundEntities,
DictionaryVersion dictionaryVersion,
long rulesVersion) {
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
@ -70,16 +71,24 @@ public class EntityLogCreatorService {
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(),
previousExistingEntityLogEntries,
entityLogEntries,
analyzeRequest.getAnalysisNumber());
return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
dictionaryVersion.getDossierTemplateVersion(),
dictionaryVersion.getDossierVersion(),
rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
return EntityLogChanges.builder()
.entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
dictionaryVersion.getDossierTemplateVersion(),
dictionaryVersion.getDossierVersion(),
rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())))
.updatedEntityLogEntries(entryChanges.updated())
.newEntityLogEntries(entryChanges.inserted())
.build();
}
@ -93,7 +102,11 @@ public class EntityLogCreatorService {
}
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, List<EntityLogEntry> newEntries, List<EntityLogEntry> updatedEntries) {
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
DictionaryVersion dictionaryVersion,
AnalyzeRequest analyzeRequest,
List<EntityLogEntry> newEntries,
List<EntityLogEntry> updatedEntries) {
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
@ -108,8 +121,9 @@ public class EntityLogCreatorService {
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document,
EntityLog entityLogWithoutEntries,
List<PrecursorEntity> notFoundEntries,
EntityLog previousEntityLog,
List<EntityLogEntry> relevantEntityLogEntries,
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
@ -121,18 +135,17 @@ public class EntityLogCreatorService {
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0))))
.collect(Collectors.toList());
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = new ArrayList<>(relevantEntityLogEntries);
previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newEntityIds));
previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithoutContainingNodeIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(),
//analyzeRequest.getFileId(),
previousEntriesFromReAnalyzedSections,
newEntityLogEntries,
analyzeRequest.getAnalysisNumber());
previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
}

37
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -87,6 +87,34 @@ public class RedactionStorageService {
}
@SneakyThrows
public void updateEntityLogWithoutEntries(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.saveEntityLogWithoutEntries(dossierId, fileId, entityLog);
}
@SneakyThrows
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.insertEntityLog(dossierId, fileId, entityLog);
}
@SneakyThrows
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
entityLogMongoService.insertEntityLogEntries(dossierId, fileId, entityLogEntries);
}
@SneakyThrows
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
entityLogMongoService.updateEntityLogEntries(dossierId, fileId, entityLogEntries);
}
@Timed("redactmanager_getImportedRedactions")
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
@ -177,6 +205,15 @@ public class RedactionStorageService {
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
}
public List<EntityLogEntry> findEntriesWithoutContainingNodeIds(String dossierId, String fileId) {
return entityLogMongoService.findAllEntityLogEntriesWithoutContainingNodeIds(dossierId, fileId);
}
public List<EntityLogEntry> findEntriesWithIds(String dossierId, String fileId, Collection<String> entryIds) {
return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
}
// !Warning! before activating redis cache you need to set

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java
View File

@ -138,7 +138,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
document,
notFoundManualEntities,
new DictionaryVersion(),
0L).getEntityLogEntry();
0L).getEntityLog().getEntityLogEntry();
assertEquals(1, redactionLogEntries.size());
assertEquals(value, redactionLogEntries.get(0).getValue());