diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 698b9f80..87a7c5a3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -1,5 +1,8 @@ package com.iqser.red.service.redaction.v1.server.service; +import static com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService.getRelevantManuallyModifiedAnnotationIds; + +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -20,6 +23,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; @@ -80,7 +84,7 @@ public class AnalyzeService { public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { long startTime = System.currentTimeMillis(); - EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + EntityLog entityLogWithoutEntries = redactionStorageService.getEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); @@ -90,25 +94,37 @@ public class AnalyzeService { log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); // not yet ready for reanalysis - if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) { + if (entityLogWithoutEntries == null || document == null || document.getNumberOfPages() == 0) { return analyze(analyzeRequest); } DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), - new DictionaryVersion(previousEntityLog.getDictionaryVersion(), - previousEntityLog.getDossierDictionaryVersion()), + new DictionaryVersion(entityLogWithoutEntries.getDictionaryVersion(), + entityLogWithoutEntries.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement, importedRedactions); + Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); + + List relevantEntityLogEntries = redactionStorageService.findEntriesWithContainingNodeIds(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + relevantManuallyModifiedAnnotationIds); + Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, + document, + dictionaryIncrement, + importedRedactions, + relevantEntityLogEntries, + relevantManuallyModifiedAnnotationIds); + List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (sectionsToReAnalyse.isEmpty()) { - EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog, + // todo: 8702 - this feels wrong + EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryIncrement.getDictionaryVersion(), analyzeRequest, - false); + new ArrayList<>(), new ArrayList<>()); return finalizeAnalysis(analyzeRequest, startTime, @@ -154,7 +170,7 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, document, notFoundManualOrImportedEntries, - previousEntityLog, + entityLogWithoutEntries, sectionsToReanalyseIds, dictionary.getVersion()); @@ -228,7 +244,7 @@ public class AnalyzeService { return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, - new EntityLogChanges(entityLog, false), + EntityLogChanges.builder().entityLog(entityLog).build(), document, document.getNumberOfPages(), dictionary.getVersion(), @@ -248,7 +264,7 @@ public class AnalyzeService { Set addedFileAttributes) { EntityLog entityLog = entityLogChanges.getEntityLog(); - //redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog()); + redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog()); log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -317,12 +333,18 @@ public class AnalyzeService { private Set getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, - EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement, - ImportedRedactions importedRedactions) { + ImportedRedactions importedRedactions, + List entityLogEntries, + Set relevantManuallyModifiedAnnotationIds) { - return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest, importedRedactions); + return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, + document, + analyzeRequest, + importedRedactions, + entityLogEntries, + relevantManuallyModifiedAnnotationIds); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index ecedefc5..c6da211a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -93,7 +93,7 @@ public class EntityLogCreatorService { } - public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, boolean hasChanges) { + public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, List newEntries, List updatedEntries) { List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); @@ -102,7 +102,7 @@ public class EntityLogCreatorService { entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); entityLog.setAnalysisNumber(analyzeRequest.getAnalysisNumber()); - return new EntityLogChanges(entityLog, hasChanges); + return EntityLogChanges.builder().entityLog(entityLog).newEntityLogEntries(newEntries).updatedEntityLogEntries(updatedEntries).build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index a78360b8..aabccc20 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -11,7 +11,6 @@ import java.util.stream.Stream; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; @@ -44,23 +43,17 @@ public class SectionFinderService { @Timed("redactmanager_findSectionsToReanalyse") public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, - EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest, - ImportedRedactions importedRedactions) { + ImportedRedactions importedRedactions, + List entityLogEntries, + Set relevantManuallyModifiedAnnotationIds) { long start = System.currentTimeMillis(); - Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set sectionsToReanalyse = new HashSet<>(); - for (EntityLogEntry entry : entityLog.getEntityLogEntry()) { - if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { - if (entry.getContainingNodeId().isEmpty()) { - continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis. - } - sectionsToReanalyse.add(entry.getContainingNodeId() - .get(0)); - } - } + + entityLogEntries.forEach(entityLogEntry -> sectionsToReanalyse.add(entityLogEntry.getContainingNodeId() + .get(0))); var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues() .stream() @@ -133,7 +126,7 @@ public class SectionFinderService { } - private static Set getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) { + public static Set getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) { if (manualRedactions == null) { return new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 5af34a89..572ad72f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.storage; import java.io.File; import java.io.FileInputStream; import java.io.InputStream; +import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -10,6 +11,7 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -142,8 +144,8 @@ public class RedactionStorageService { public EntityLog getEntityLog(String dossierId, String fileId) { try { - //EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class); - EntityLog entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId).orElseThrow(() -> new StorageObjectDoesNotExist("")); + EntityLog entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId) + .orElseThrow(() -> new StorageObjectDoesNotExist("")); entityLog.setEntityLogEntry(entityLog.getEntityLogEntry() .stream() .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty())) @@ -157,6 +159,26 @@ public class RedactionStorageService { } + @Timed("redactmanager_getRedactionLog") + public EntityLog getEntityLogWithoutEntries(String dossierId, String fileId) { + + try { + return entityLogMongoService.findEntityLogWithoutEntries(dossierId, fileId) + .orElseThrow(() -> new StorageObjectDoesNotExist("")); + } catch (StorageObjectDoesNotExist e) { + log.debug("EntityLog not available."); + return null; + } + + } + + + public List findEntriesWithContainingNodeIds(String dossierId, String fileId, Collection entryIds) { + + return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds); + } + + // !Warning! before activating redis cache you need to set // -Dio.netty.noPreferDirect=true -XX:MaxDirectMemorySize=1000M // Jvm args to the largest document data size we want to process. for 4443 pages file that was 500mb.