From 93e84fe166baa5c98a4dfd53ab815ecb67865e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Thu, 28 Mar 2024 09:57:27 +0100 Subject: [PATCH] RED-7384: Migration fixes --- ...asMigrationManualChangesUpdateService.java | 151 ++++++++++++++++++ .../migration/SaasMigrationService.java | 5 +- .../ManualRedactionProviderService.java | 14 -- .../AddRedactionPersistenceService.java | 2 +- .../RemoveRedactionPersistenceService.java | 9 ++ .../ResizeRedactionPersistenceService.java | 6 + .../repository/FileRepository.java | 117 +++++++++++--- .../RedactionAnalysisResponseReceiver.java | 2 +- 8 files changed, 266 insertions(+), 40 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java new file mode 100644 index 000000000..304b81bae --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java @@ -0,0 +1,151 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; + +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; +import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class SaasMigrationManualChangesUpdateService { + + private final AddRedactionPersistenceService addRedactionPersistenceService; + private final RemoveRedactionPersistenceService removeRedactionPersistenceService; + private final ResizeRedactionPersistenceService resizeRedactionPersistenceService; + + private final HashFunction hashFunction = Hashing.murmur3_128(); + + + public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId) { + + var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly()); + for (var unprocessedManualAdd : unprocessedManualAdds) { + + if (!unprocessedManualAdd.getDictionaryEntryType().equals(DictionaryEntryType.ENTRY)) { + continue; + } + + if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) { + // copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table. + // It has no functionality, its only there, such that the UI can show a pending change. + ManualRedactionEntryEntity pendingDictAdd = ManualRedactionEntryEntity.builder() + .id(buildSecondaryId(unprocessedManualAdd.getId(), fileId)) + .user(unprocessedManualAdd.getUser()) + .typeId(unprocessedManualAdd.getTypeId()) + .value(unprocessedManualAdd.getValue()) + .reason(unprocessedManualAdd.getReason()) + .legalBasis(unprocessedManualAdd.getLegalBasis()) + .section(unprocessedManualAdd.getSection()) + .rectangle(unprocessedManualAdd.isRectangle()) + .addToDictionary(unprocessedManualAdd.isAddToDictionary()) + .addToAllDossiers(unprocessedManualAdd.isAddToAllDossiers()) + .dictionaryEntryType(DictionaryEntryType.ENTRY) + .requestDate(unprocessedManualAdd.getRequestDate()) + .positions(new ArrayList<>(unprocessedManualAdd.getPositions())) // copy to new List + .fileStatus(unprocessedManualAdd.getFileStatus()) + .textBefore(unprocessedManualAdd.getTextBefore()) + .textAfter(unprocessedManualAdd.getTextAfter()) + .sourceId(unprocessedManualAdd.getSourceId()) + .typeIdsOfModifiedDictionaries(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries()) + .build(); + + addRedactionPersistenceService.updateOrCreate(pendingDictAdd); + + // change existing dict add to unprocessed manual add. ID must match with prior entry, such that other unprocessed manual changes may be applied to it. + unprocessedManualAdd.setAddToDictionary(false); + unprocessedManualAdd.setAddToAllDossiers(false); + unprocessedManualAdd.setLegalBasis(""); + unprocessedManualAdd.setTypeIdsOfModifiedDictionaries(Collections.emptySet()); + unprocessedManualAdd.setDictionaryEntryType(null); + + addRedactionPersistenceService.updateOrCreate(unprocessedManualAdd); + } + } + } + + + public void convertUnprocessedRemoveFromDictionariesToLocalChanges(String fileId) { + + var unprocessedManualRemoves = removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly()); + for (var unprocessedManualRemove : unprocessedManualRemoves) { + if (unprocessedManualRemove.isRemoveFromDictionary() || unprocessedManualRemove.isRemoveFromAllDossiers()) { + + IdRemovalEntity pendingDictRemoval = IdRemovalEntity.builder() + .id(buildSecondaryId(unprocessedManualRemove.getId(), fileId)) + .user(unprocessedManualRemove.getUser()) + .removeFromDictionary(unprocessedManualRemove.isRemoveFromDictionary()) + .removeFromAllDossiers(unprocessedManualRemove.isRemoveFromAllDossiers()) + .requestDate(unprocessedManualRemove.getRequestDate()) + .page(unprocessedManualRemove.getPage()) + .fileStatus(unprocessedManualRemove.getFileStatus()) + .typeIdsOfModifiedDictionaries(unprocessedManualRemove.getTypeIdsOfModifiedDictionaries()) + .build(); + + removeRedactionPersistenceService.updateOrCreate(pendingDictRemoval); + + unprocessedManualRemove.setRemoveFromDictionary(false); + unprocessedManualRemove.setRemoveFromAllDossiers(false); + unprocessedManualRemove.setTypeIdsOfModifiedDictionaries(Collections.emptySet()); + + removeRedactionPersistenceService.updateOrCreate(unprocessedManualRemove); + } + } + } + + + public void convertUnprocessedResizeWithDictionariesToLocalChanges(String fileId) { + + var unprocessedManualResizes = resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly()); + for (var unprocessedManualResize : unprocessedManualResizes) { + if (unprocessedManualResize.getUpdateDictionary() || unprocessedManualResize.isAddToAllDossiers()) { + + ManualResizeRedactionEntity pendingDictResize = ManualResizeRedactionEntity.builder() + .id(buildSecondaryId(unprocessedManualResize.getId(), fileId)) + .user(unprocessedManualResize.getUser()) + .requestDate(unprocessedManualResize.getRequestDate()) + .page(unprocessedManualResize.getPage()) + .value(unprocessedManualResize.getValue()) + .updateDictionary(unprocessedManualResize.getUpdateDictionary()) + .fileStatus(unprocessedManualResize.getFileStatus()) + .positions(new ArrayList<>(unprocessedManualResize.getPositions())) + .textBefore(unprocessedManualResize.getTextBefore()) + .textAfter(unprocessedManualResize.getTextAfter()) + .addToAllDossiers(unprocessedManualResize.isAddToAllDossiers()) + .typeIdsOfModifiedDictionaries(unprocessedManualResize.getTypeIdsOfModifiedDictionaries()) + .build(); + + resizeRedactionPersistenceService.updateOrCreate(pendingDictResize); + + unprocessedManualResize.setUpdateDictionary(false); + unprocessedManualResize.setAddToAllDossiers(false); + unprocessedManualResize.setTypeIdsOfModifiedDictionaries(Collections.emptySet()); + + resizeRedactionPersistenceService.updateOrCreate(unprocessedManualResize); + } + } + } + + + private AnnotationEntityId buildSecondaryId(AnnotationEntityId annotationEntityId, String fileId) { + + return new AnnotationEntityId(hashFunction.hashString(annotationEntityId.getAnnotationId(), StandardCharsets.UTF_8).toString(), fileId); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 295f23f12..ce75b1f73 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -71,6 +71,7 @@ public class SaasMigrationService implements TenantSyncService { ManualRedactionService manualRedactionService; CommentService commentService; RankDeDuplicationService rankDeDuplicationService; + SaasMigrationManualChangesUpdateService saasMigrationManualChangesUpdateService; @Override @@ -159,7 +160,9 @@ public class SaasMigrationService implements TenantSyncService { saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED); if (fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { - manualRedactionProviderService.convertUnprocessedAddToDictionariesToLocalChanges(fileId); + saasMigrationManualChangesUpdateService.convertUnprocessedAddToDictionariesToLocalChanges(fileId); + saasMigrationManualChangesUpdateService.convertUnprocessedRemoveFromDictionariesToLocalChanges(fileId); + saasMigrationManualChangesUpdateService.convertUnprocessedResizeWithDictionariesToLocalChanges(fileId); } try { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java index 0e6edbe61..a8b8343a5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java @@ -47,20 +47,6 @@ public class ManualRedactionProviderService { private final ResizeRedactionPersistenceService resizeRedactionPersistenceService; - - public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId){ - var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly()); - for (var unprocessedManualAdd : unprocessedManualAdds){ - if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()){ - unprocessedManualAdd.setAddToDictionary(false); - unprocessedManualAdd.setAddToAllDossiers(false); - unprocessedManualAdd.setLegalBasis(""); - addRedactionPersistenceService.update(unprocessedManualAdd); - } - } - } - - @Transactional public ManualRedactions getManualRedactions(String fileId, ManualChangesQueryOptions options) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java index a2b6c93bf..83972ec30 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java @@ -115,7 +115,7 @@ public class AddRedactionPersistenceService { @Transactional - public void update(ManualRedactionEntryEntity manualRedactionEntry) { + public void updateOrCreate(ManualRedactionEntryEntity manualRedactionEntry) { manualRedactionRepository.saveAndFlush(manualRedactionEntry); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java index 119928129..5016769e1 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java @@ -65,11 +65,13 @@ public class RemoveRedactionPersistenceService { return removeRedactionRepository.findAll(); } + public List findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) { return removeRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges()); } + @Transactional public void hardDelete(String fileId, String annotationId) { @@ -110,4 +112,11 @@ public class RemoveRedactionPersistenceService { removeRedactionRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } + + public void updateOrCreate(IdRemovalEntity unprocessedManualRemove) { + + removeRedactionRepository.saveAndFlush(unprocessedManualRemove); + + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java index b81031a1e..8a72fe006 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java @@ -123,4 +123,10 @@ public class ResizeRedactionPersistenceService { return resizeRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges()); } + + public void updateOrCreate(ManualResizeRedactionEntity unprocessedManualResize) { + + resizeRedactionRepository.saveAndFlush(unprocessedManualResize); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 4bd687674..fc1e6e484 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -31,7 +31,11 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + " e.lastUpdated = :lastUpdated " + " where e.id =:fileId") + @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + + " e.lastUpdated = :lastUpdated " + + " where e.id =:fileId") void updateFlags(@Param("fileId") String fileId, @Param("lastUpdated") OffsetDateTime lastUpdated, @Param("hasRedactions") boolean hasRedactions, @@ -43,7 +47,12 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, " + + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + + "where f.id = :fileId") void updateProcessingStatus(@Param("fileId") String fileId, @Param("numberOfPages") int numberOfPages, @Param("processingStatus") ProcessingStatus processingStatus, @@ -61,7 +70,8 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId") + @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId") void updateWorkflowStatus(@Param("fileId") String fileId, @Param("workflowStatus") WorkflowStatus workflowStatus, @Param("lastUpdated") OffsetDateTime lastUpdated, @@ -83,7 +93,9 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + " f.hasHighlights = :hasHighlights, f.processingErrorCounter = :processingErrorCounter " + " where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + + " f.hasHighlights = :hasHighlights, f.processingErrorCounter = :processingErrorCounter " + + " where f.id = :fileId") void updateProcessingStatus(@Param("fileId") String fileId, @Param("processingStatus") ProcessingStatus processingStatus, @Param("lastUpdated") OffsetDateTime lastUpdated, @@ -92,7 +104,8 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + + "where f.id = :fileId") void updateProcessingStatus(@Param("fileId") String fileId, @Param("processingStatus") ProcessingStatus processingStatus, @Param("lastUpdated") OffsetDateTime lastUpdated, @@ -109,7 +122,8 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true, flushAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + + "where f.id = :fileId") void updateProcessingStatus(@Param("fileId") String fileId, @Param("processingStatus") ProcessingStatus processingStatus, @Param("lastUpdated") OffsetDateTime lastUpdated, @@ -118,7 +132,8 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId") void setUpdateStatusIndexingSuccessful(@Param("fileId") String fileId, @Param("processingStatus") ProcessingStatus processingStatus, @Param("lastUpdated") OffsetDateTime lastUpdated, @@ -155,7 +170,13 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.hardDeletedTime = :hardDeletedTime, " + "f.deleted = case " + " when f.deleted is null then :deleted " + " when f.deleted is not null then f.deleted " + "end " + "where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + + "f.hardDeletedTime = :hardDeletedTime, " + + "f.deleted = case " + + " when f.deleted is null then :deleted " + + " when f.deleted is not null then f.deleted " + + "end " + + "where f.id = :fileId") int setHardDelete(@Param("fileId") String fileId, @Param("processingStatus") ProcessingStatus processingStatus, @Param("lastUpdated") OffsetDateTime lastUpdated, @@ -185,7 +206,18 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " + "f.fileManipulationDate = :lastUploaded, " + "f.ocrEndTime = null, f.ocrStartTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, " + "f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, " + "f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, " + "f.processingErrorCounter = 0, f.errorCause = null, f.errorQueue = null, f.errorService = null, " + "f.errorTimestamp = null where f.id = :fileId") + @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " + + "f.fileManipulationDate = :lastUploaded, " + + "f.ocrEndTime = null, f.ocrStartTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, " + + "f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, " + + "f.redactionModificationDate = null, " + + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, " + + "f.processingErrorCounter = 0, f.errorCause = null, f.errorQueue = null, f.errorService = null, " + + "f.errorTimestamp = null where f.id = :fileId") int overwriteFile(@Param("fileId") String fileId, @Param("filename") String filename, @Param("uploader") String uploader, @@ -196,7 +228,16 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastProcessed = null," + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0, " + "f.ocrStartTime = null, f.ocrEndTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, " + "f.errorCause = null, f.errorQueue = null, f.errorService = null, f.errorTimestamp = null " + "where f.id = :fileId") + @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + + "f.lastProcessed = null," + + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0, " + + "f.ocrStartTime = null, f.ocrEndTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, " + + "f.errorCause = null, f.errorQueue = null, f.errorService = null, f.errorTimestamp = null " + + "where f.id = :fileId") int overwriteFileAndKeepManualRedactions(@Param("fileId") String fileId, @Param("filename") String filename, @Param("uploader") String uploader, @@ -205,11 +246,17 @@ public interface FileRepository extends JpaRepository { @Param("lastUpdated") OffsetDateTime lastUpdated); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId " + "where d.dossierTemplateId = :dossierTemplateId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId " + + "where d.dossierTemplateId = :dossierTemplateId" + + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") int countSoftDeletedFilesPerDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId " + "where d.id = :dossierId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null))") + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId " + + "where d.id = :dossierId" + + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + + " (d.softDeletedTime is not null and d.hardDeletedTime is null))") int countSoftDeletedFilesPerDossierId(@Param("dossierId") String dossierId); @@ -238,7 +285,11 @@ public interface FileRepository extends JpaRepository { @Param("lastUpdated") OffsetDateTime lastUpdated); - @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId " + "where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )" + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries") + @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId " + + "where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )" + + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries") List getAllRelevantStatusesForReanalysisScheduler(@Param("maxRetries") int maxRetries); @@ -251,25 +302,45 @@ public interface FileRepository extends JpaRepository { List getSoftDeletedFiles(@Param("dossierIds") List dossierIds); - @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.processingStatus ") + @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + + "inner join DossierEntity d on d.id = f.dossierId " + + "where f.deleted is null and f.hardDeletedTime is null " + + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + + "and d.dossierTemplateId = :dossierTemplateId " + + "group by f.processingStatus ") List countFilesByProcessingStatus(@Param("dossierTemplateId") String dossierTemplateId); - @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.workflowStatus ") + @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + + "inner join DossierEntity d on d.id = f.dossierId " + + "where f.deleted is null and f.hardDeletedTime is null " + + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + + "and d.dossierTemplateId = :dossierTemplateId " + + "group by f.workflowStatus ") List countFilesByWorkflowStatus(@Param("dossierTemplateId") String dossierTemplateId); - @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, " + "COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + "from file join dossier on file.dossier_id = dossier.id " + "where file.deleted is null and file.hard_deleted_time is null " + "and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true) + @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, " + + "COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + + "from file join dossier on file.dossier_id = dossier.id " + + "where file.deleted is null and file.hard_deleted_time is null " + + "and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true) FilePageCountsProjection countPages(@Param("dossierTemplateId") String dossierTemplateId); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + "f.hardDeletedTime is null and f.deleted is null and " + "d.dossierTemplateId = :dossierTemplateId and " + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null") + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + + "f.hardDeletedTime is null and f.deleted is null and " + + "d.dossierTemplateId = :dossierTemplateId and " + + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null") int countActiveFiles(@Param("dossierTemplateId") String dossierTemplateId); @Transactional @Modifying(clearAutomatically = true) - @Query(value = "update FileEntity f set f.numberOfOCRedPages = :numberOfOCRedPages, " + "f.numberOfPagesToOCR = :numberOfPagesToOCR, f.ocrEndTime = :ocrEndTime, " + "f.lastUpdated = :lastUpdated where f.id = :fileId") + @Query(value = "update FileEntity f set f.numberOfOCRedPages = :numberOfOCRedPages, " + + "f.numberOfPagesToOCR = :numberOfPagesToOCR, f.ocrEndTime = :ocrEndTime, " + + "f.lastUpdated = :lastUpdated where f.id = :fileId") void updateOCRStatus(@Param("fileId") String fileId, @Param("numberOfPagesToOCR") int numberOfPagesToOCR, @Param("numberOfOCRedPages") int numberOfOCRedPages, @@ -300,11 +371,11 @@ public interface FileRepository extends JpaRepository { * @return a List of Tuples of fileIds and dossierIds, where a flag calculation is necessary */ @Query(""" - select distinct f.id, f.dossierId - from FileEntity f - left join ViewedPageEntity v on f.id = v.file.id and v.id.userId = f.assignee - where (f.lastFlagCalculation is NULL and f.lastProcessed is not NULL) or f.lastManualChangeDate > f.lastFlagCalculation or f.lastProcessed > f.lastFlagCalculation or f.lastFlagCalculation < v.viewedTime - """) + select distinct f.id, f.dossierId + from FileEntity f + left join ViewedPageEntity v on f.id = v.file.id and v.id.userId = f.assignee + where (f.lastFlagCalculation is NULL and f.lastProcessed is not NULL) or f.lastManualChangeDate > f.lastFlagCalculation or f.lastProcessed > f.lastFlagCalculation or f.lastFlagCalculation < v.viewedTime + """) List getFileIdentifiersWhereAnalysisFlagCalculationIsRequired(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionAnalysisResponseReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionAnalysisResponseReceiver.java index 9d9d50999..467cfe8e9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionAnalysisResponseReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionAnalysisResponseReceiver.java @@ -57,7 +57,7 @@ public class RedactionAnalysisResponseReceiver { if (StringUtils.isNullOrEmpty(manualRedactionEntry.getSection()) || (!StringUtils.isNullOrEmpty(unprocessedManualEntity.getSection()) && !manualRedactionEntry.getSection().equals(unprocessedManualEntity.getSection()))) { manualRedactionEntry.setSection(unprocessedManualEntity.getSection()); } - addRedactionPersistenceService.update(manualRedactionEntry); + addRedactionPersistenceService.updateOrCreate(manualRedactionEntry); } Optional optionalManualResizeRedactionEntity = resizeRedactionPersistenceService.findResizeRedactionById(fileId, unprocessedManualEntity.getAnnotationId());