diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 87a7c5a3..60dfa597 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -124,7 +124,8 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryIncrement.getDictionaryVersion(), analyzeRequest, - new ArrayList<>(), new ArrayList<>()); + new ArrayList<>(), + new ArrayList<>()); return finalizeAnalysis(analyzeRequest, startTime, @@ -169,8 +170,9 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, document, - notFoundManualOrImportedEntries, entityLogWithoutEntries, + notFoundManualOrImportedEntries, + relevantEntityLogEntries, sectionsToReanalyseIds, dictionary.getVersion()); @@ -233,18 +235,18 @@ public class AnalyzeService { nerEntities); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest, - document, - notFoundManualOrImportedEntries, - dictionary.getVersion(), - kieWrapperEntityRules.rulesVersion()); + EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, + document, + notFoundManualOrImportedEntries, + dictionary.getVersion(), + kieWrapperEntityRules.rulesVersion()); - notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries); + notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries); return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, - EntityLogChanges.builder().entityLog(entityLog).build(), + entityLogChanges, document, document.getNumberOfPages(), dictionary.getVersion(), @@ -265,10 +267,23 @@ public class AnalyzeService { EntityLog entityLog = entityLogChanges.getEntityLog(); - redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog()); + //hotfix for tests (todo 8702) + if (entityLog.getAnalysisNumber() <= 0 && !redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) { + redactionStorageService.insertEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + + } else { + redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + + if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) { + redactionStorageService.insertEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries()); + } + if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) { + redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries()); + } + } log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - if (entityLogChanges.isHasChanges() || !isReanalysis) { + if (entityLogChanges.hasChanges() || !isReanalysis) { computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion); } @@ -283,7 +298,7 @@ public class AnalyzeService { .fileId(analyzeRequest.getFileId()) .duration(duration) .numberOfPages(numberOfPages) - .hasUpdates(entityLogChanges.isHasChanges()) + .hasUpdates(entityLogChanges.hasChanges()) .analysisVersion(redactionServiceSettings.getAnalysisVersion()) .analysisNumber(analyzeRequest.getAnalysisNumber()) .rulesVersion(entityLog.getRulesVersion()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index ee0fb76d..43e0e847 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -32,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import lombok.AccessLevel; @@ -60,11 +61,11 @@ public class EntityLogCreatorService { } - public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, - Document document, - List notFoundEntities, - DictionaryVersion dictionaryVersion, - long rulesVersion) { + public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundEntities, + DictionaryVersion dictionaryVersion, + long rulesVersion) { List entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities); @@ -72,16 +73,24 @@ public class EntityLogCreatorService { List previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber()); + EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(), + //analyzeRequest.getFileId(), + previousExistingEntityLogEntries, + entityLogEntries, + analyzeRequest.getAnalysisNumber()); - return new EntityLog(redactionServiceSettings.getAnalysisVersion(), - analyzeRequest.getAnalysisNumber(), - entityLogEntries, - toEntityLogLegalBasis(legalBasis), - dictionaryVersion.getDossierTemplateVersion(), - dictionaryVersion.getDossierVersion(), - rulesVersion, - legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + return EntityLogChanges.builder() + .entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(), + analyzeRequest.getAnalysisNumber(), + entityLogEntries, + toEntityLogLegalBasis(legalBasis), + dictionaryVersion.getDossierTemplateVersion(), + dictionaryVersion.getDossierVersion(), + rulesVersion, + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()))) + .updatedEntityLogEntries(entryChanges.updated()) + .newEntityLogEntries(entryChanges.inserted()) + .build(); } @@ -95,7 +104,11 @@ public class EntityLogCreatorService { } - public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, List newEntries, List updatedEntries) { + public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, + DictionaryVersion dictionaryVersion, + AnalyzeRequest analyzeRequest, + List newEntries, + List updatedEntries) { List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); @@ -110,8 +123,9 @@ public class EntityLogCreatorService { public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, Document document, + EntityLog entityLogWithoutEntries, List notFoundEntries, - EntityLog previousEntityLog, + List relevantEntityLogEntries, Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { @@ -123,18 +137,17 @@ public class EntityLogCreatorService { .map(EntityLogEntry::getId) .collect(Collectors.toSet()); - List previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() - .stream() - .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() - .get(0)))) - .collect(Collectors.toList()); - previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections); + List previousEntriesFromReAnalyzedSections = new ArrayList<>(relevantEntityLogEntries); + previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newEntityIds)); + previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithoutContainingNodeIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); + EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(), + //analyzeRequest.getFileId(), + previousEntriesFromReAnalyzedSections, + newEntityLogEntries, + analyzeRequest.getAnalysisNumber()); - previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); - - return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges); + return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 572ad72f..99e51913 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -87,6 +87,34 @@ public class RedactionStorageService { } + @SneakyThrows + public void updateEntityLogWithoutEntries(String dossierId, String fileId, EntityLog entityLog) { + + entityLogMongoService.saveEntityLogWithoutEntries(dossierId, fileId, entityLog); + } + + + @SneakyThrows + public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) { + + entityLogMongoService.insertEntityLog(dossierId, fileId, entityLog); + } + + + @SneakyThrows + public void insertEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + entityLogMongoService.insertEntityLogEntries(dossierId, fileId, entityLogEntries); + } + + + @SneakyThrows + public void updateEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + entityLogMongoService.updateEntityLogEntries(dossierId, fileId, entityLogEntries); + } + + @Timed("redactmanager_getImportedRedactions") public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { @@ -177,6 +205,15 @@ public class RedactionStorageService { return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds); } + public List findEntriesWithoutContainingNodeIds(String dossierId, String fileId) { + + return entityLogMongoService.findAllEntityLogEntriesWithoutContainingNodeIds(dossierId, fileId); + } + + public List findEntriesWithIds(String dossierId, String fileId, Collection entryIds) { + + return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds); + } // !Warning! before activating redis cache you need to set diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java index b6a1efcb..81acfe52 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java @@ -138,7 +138,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { document, notFoundManualEntities, new DictionaryVersion(), - 0L).getEntityLogEntry(); + 0L).getEntityLog().getEntityLogEntry(); assertEquals(1, redactionLogEntries.size()); assertEquals(value, redactionLogEntries.get(0).getValue());