Pull request #179: Analysis number in redaction log m

Merge in RED/persistence-service from analysis-number-in-redaction-log-m to master

* commit '7ff197d73029a200aea64e7898d28763cbd71373':
  analysis version in redactionlog and changes
  analysis version in redactionlog and changes

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -53,14 +53,14 @@ public class FileStatusPersistenceService {
     @Transactional
     public void updateWorkflowStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion,
                                      long legalBasisVersion, long duration, long dossierDictionaryVersion,
-                                     int analysisVersion) {
+                                     int analysisVersion, int analysisNumber) {
         if (isFileDeleted(fileId)) {
             return;
         }
         fileRepository.updateWorkflowStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED,
                 dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion,
                 analysisVersion, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -38,12 +38,12 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " +
             "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " +
             "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " +
-            "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = f.numberOfAnalyses + 1, f.lastUpdated = :lastUpdated, " +
+            "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " +
             "f.lastProcessed = :lastProcessed " +
             "where f.id = :fileId")
     void updateWorkflowStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion,
                               long rulesVersion, long legalBasisVersion, long analysisDuration, long dossierDictionaryVersion,
-                              int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed);
+                              int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber);
 
     @Modifying
     @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " +
@@ -109,7 +109,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " +
             "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " +
             "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " +
-            "f.assignee = null, f.approvalDate = null, f.lastManualRedaction = null, " +
+            "f.assignee = null, f.approvalDate = null, f.lastManualRedaction = null, f.numberOfAnalyses = 0, " +
             "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " +
             "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " +
             "f.deleted = null, f.hardDeletedTime = null " +

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java

@@ -1,127 +1,42 @@
 package com.iqser.red.service.peristence.v1.server.migration;
 
-import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
-import static java.util.stream.Collectors.toList;
-
-import java.time.OffsetDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.peristence.v1.server.exception.MigrationException;
-import com.iqser.red.service.peristence.v1.server.migration.model.CommentRow;
+import com.iqser.red.service.peristence.v1.server.migration.model.*;
-import com.iqser.red.service.peristence.v1.server.migration.model.ConfigurationRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.DigitalSignatureModel;
-import com.iqser.red.service.peristence.v1.server.migration.model.DossierAttributeRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.DossierAttributesConfig;
-import com.iqser.red.service.peristence.v1.server.migration.model.EntryRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.FileAttributesConfig;
-import com.iqser.red.service.peristence.v1.server.migration.model.IdRemovalRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.ManualForceRedactionRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.ManualImageRecategorizationRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.ManualLegalBasisChangeRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.ManualRedactionEntryRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationAuditModel;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationComment;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDossier;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDossierTemplate;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDownloadStatus;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationFileStatus;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationNotification;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationRedactionLog;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationRedactionLogEntry;
-import com.iqser.red.service.peristence.v1.server.migration.model.MigrationViewedPage;
-import com.iqser.red.service.peristence.v1.server.migration.model.NewIdForDossierTemplate;
-import com.iqser.red.service.peristence.v1.server.migration.model.Status;
-import com.iqser.red.service.peristence.v1.server.migration.model.VersionRow;
-import com.iqser.red.service.peristence.v1.server.migration.model.VersionType;
 import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
 import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.audit.AuditEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.*;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DigitalSignatureEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisMappingEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.notification.NotificationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.service.EncryptionDecryptionService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ColorsRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeConfigRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ForceRedactionRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ImageRecategorizationRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisChangeRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisMappingRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ManualRedactionRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RemoveRedactionRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RuleSetRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ViewedPagesRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
-import com.iqser.red.service.redaction.v1.model.Change;
+import com.iqser.red.service.redaction.v1.model.*;
-import com.iqser.red.service.redaction.v1.model.ChangeType;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
+import static java.util.stream.Collectors.toList;
 
 @SuppressWarnings("PMD")
 @Slf4j
@@ -264,41 +179,42 @@ public class MigrationService {
 
         List<FileEntity> files = fileRepository.findAll();
         files.stream().filter(file -> file.getHardDeletedTime() == null).forEach(file -> {
-            migrateRedactionLog(file.getDossierId(), file.getId(), file.getLastProcessed());
+            migrateRedactionLog(file);
         });
     }
 
 
     @SneakyThrows
-    private void migrateRedactionLog(String dossierId, String fileId, OffsetDateTime lastProcessed) {
+    private void migrateRedactionLog(FileEntity file) {
 
         try {
-            byte[] oldRedactionLogBytes = fileManagementStorageService.getStoredObjectBytes(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
+            byte[] oldRedactionLogBytes = fileManagementStorageService.getStoredObjectBytes(StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG));
 
             if (oldRedactionLogBytes == null) {
-                log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", dossierId, fileId);
+                log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", file.getDossierId(), file.getId());
                 return;
             }
             var oldRedactionLog = objectMapper.readValue(oldRedactionLogBytes, MigrationRedactionLog.class);
 
-            RedactionLog redactionLog = new RedactionLog(oldRedactionLog.getAnalysisVersion(), oldRedactionLog.getRedactionLogEntry()
+            RedactionLog redactionLog = new RedactionLog(oldRedactionLog.getAnalysisVersion(),
+                    file.getNumberOfAnalyses(), oldRedactionLog.getRedactionLogEntry()
                     .stream()
-                    .map(r -> convertRedactionLogEntry(r, fileId, lastProcessed))
+                    .map(r -> convertRedactionLogEntry(r, file.getId(), file.getLastProcessed(),file.getNumberOfAnalyses()))
                     .collect(toList()), oldRedactionLog.getLegalBasis(), oldRedactionLog.getDictionaryVersion(), oldRedactionLog
                     .getDossierDictionaryVersion(), oldRedactionLog.getRulesVersion(), oldRedactionLog.getLegalBasisVersion());
 
-            storageService.storeObject("backup/" + StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), objectMapper
+            storageService.storeObject("backup/" + StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG), objectMapper
                     .writeValueAsBytes(oldRedactionLogBytes));
-            storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), objectMapper
+            storageService.storeObject(StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG), objectMapper
                     .writeValueAsBytes(redactionLog));
         } catch (StorageObjectDoesNotExist e) {
-            log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", dossierId, fileId);
+            log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", file.getDossierId(), file.getId());
         }
     }
 
 
     private RedactionLogEntry convertRedactionLogEntry(MigrationRedactionLogEntry oldRedactionLogEntry, String fileId,
-                                                       OffsetDateTime lastProcessed) {
+                                                       OffsetDateTime lastProcessed, int analysisNumber) {
 
         return RedactionLogEntry.builder()
                 .id(oldRedactionLogEntry.getId())
@@ -335,7 +251,7 @@ public class MigrationService {
                 .excluded(oldRedactionLogEntry.isExcluded())
                 .recategorizationType(oldRedactionLogEntry.getRecategorizationType())
                 .legalBasisChangeValue(oldRedactionLogEntry.getLegalBasisChangeValue())
-                .changes(List.of(new Change(ChangeType.ADDED, lastProcessed)))
+                .changes(List.of(new Change(analysisNumber, ChangeType.ADDED, lastProcessed)))
                 .engines(getEngine(oldRedactionLogEntry))
                 .reference(null)
                 .rectangle(oldRedactionLogEntry.isRectangle())
@@ -346,7 +262,7 @@ public class MigrationService {
     private Set<Engine> getEngine(MigrationRedactionLogEntry oldRedactionLogEntry) {
 
         if (oldRedactionLogEntry.isManual()) {
-            return new HashSet<Engine>();
+            return new HashSet<>();
         }
         if (oldRedactionLogEntry.isDictionaryEntry() || oldRedactionLogEntry.isDossierDictionaryEntry()) {
             return Set.of(Engine.DICTIONARY);

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java

@@ -101,7 +101,11 @@ public class FileStatusService {
 
     public void setStatusSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) {
 
-        fileStatusPersistenceService.updateWorkflowStatus(fileId, analyzeResult.getNumberOfPages(), analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion());
+        fileStatusPersistenceService.updateWorkflowStatus(fileId, analyzeResult.getNumberOfPages(),
+                analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(),
+                analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(),
+                analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(),
+                analyzeResult.getAnalysisNumber());
     }
 
 
@@ -267,6 +271,7 @@ public class FileStatusService {
         var analyseRequest = AnalyzeRequest.builder()
                 .messageType(reanalyse ? MessageType.REANALYSE : MessageType.FULL_ANALYSE)
                 .dossierId(dossierId)
+                .analysisNumber(fileStatus.getNumberOfAnalyses()+1)
                 .sectionsToReanalyse(sectionsToReanalyse)
                 .fileId(fileId)
                 .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java

@@ -58,7 +58,8 @@ public class FileTesterAndProvider {
 
         assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isGreaterThanOrEqualTo(1);
 
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(new RedactionLog(1, List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()),
+        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(new RedactionLog(1, 1,
+                List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()),
                 null, 0, 0, 0, 0)));
         fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, objectMapper.writeValueAsBytes(new SectionGrid()));
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java

@@ -129,7 +129,8 @@ public abstract class AbstractPersistenceServerServiceTest {
 
         when(redactionClient.annotate(Mockito.any())).thenReturn(new AnnotateResponse("document".getBytes()));
         when(redactionClient.sections(Mockito.any())).thenReturn(new RedactionResult());
-        when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(new RedactionLog(1, Lists.newArrayList(), null, 0, 0, 0, 0));
+        when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(new RedactionLog(1, 1,
+                Lists.newArrayList(), null, 0, 0, 0, 0));
     }
 
     @After

persistence-service-v1/pom.xml

@@ -25,7 +25,7 @@
     </modules>
 
     <properties>
-        <redaction-service.version>3.43.0</redaction-service.version>
+        <redaction-service.version>3.56.0</redaction-service.version>
         <search-service.version>2.10.0</search-service.version>
         <pdftron-redaction-service.version>3.17.0</pdftron-redaction-service.version>
         <redaction-report-service.version>3.19.0</redaction-report-service.version>