Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #179: Analysis number in redaction log m

Browse Source 
Merge in RED/persistence-service from analysis-number-in-redaction-log-m to master

* commit '7ff197d73029a200aea64e7898d28763cbd71373':
  analysis version in redactionlog and changes
  analysis version in redactionlog and changes
This commit is contained in:
Timo Bejan 2022-01-13 10:05:13 +01:00
commit bbeb29b15a
7 changed files with 45 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -53,14 +53,14 @@ public class FileStatusPersistenceService {
@Transactional
public void updateWorkflowStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion,
long legalBasisVersion, long duration, long dossierDictionaryVersion,
int analysisVersion) {
int analysisVersion, int analysisNumber) {
if (isFileDeleted(fileId)) {
return;
}
fileRepository.updateWorkflowStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED,
dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion,
analysisVersion, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -38,12 +38,12 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " +
"f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " +
"f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " +
"f.analysisVersion = :analysisVersion, f.numberOfAnalyses = f.numberOfAnalyses + 1, f.lastUpdated = :lastUpdated, " +
"f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " +
"f.lastProcessed = :lastProcessed " +
"where f.id = :fileId")
void updateWorkflowStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion,
long rulesVersion, long legalBasisVersion, long analysisDuration, long dossierDictionaryVersion,
int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed);
int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber);
@Modifying
@Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " +
@ -109,7 +109,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " +
"f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " +
"f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " +
"f.assignee = null, f.approvalDate = null, f.lastManualRedaction = null, " +
"f.assignee = null, f.approvalDate = null, f.lastManualRedaction = null, f.numberOfAnalyses = 0, " +
"f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " +
"f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " +
"f.deleted = null, f.hardDeletedTime = null " +

142
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java
View File

@ -1,127 +1,42 @@
package com.iqser.red.service.peristence.v1.server.migration;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import static java.util.stream.Collectors.toList;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.exception.MigrationException;
import com.iqser.red.service.peristence.v1.server.migration.model.CommentRow;
import com.iqser.red.service.peristence.v1.server.migration.model.ConfigurationRow;
import com.iqser.red.service.peristence.v1.server.migration.model.DigitalSignatureModel;
import com.iqser.red.service.peristence.v1.server.migration.model.DossierAttributeRow;
import com.iqser.red.service.peristence.v1.server.migration.model.DossierAttributesConfig;
import com.iqser.red.service.peristence.v1.server.migration.model.EntryRow;
import com.iqser.red.service.peristence.v1.server.migration.model.FileAttributesConfig;
import com.iqser.red.service.peristence.v1.server.migration.model.IdRemovalRow;
import com.iqser.red.service.peristence.v1.server.migration.model.ManualForceRedactionRow;
import com.iqser.red.service.peristence.v1.server.migration.model.ManualImageRecategorizationRow;
import com.iqser.red.service.peristence.v1.server.migration.model.ManualLegalBasisChangeRow;
import com.iqser.red.service.peristence.v1.server.migration.model.ManualRedactionEntryRow;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationAuditModel;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationComment;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDossier;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDossierTemplate;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDownloadStatus;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationFileStatus;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationNotification;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationRedactionLog;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationRedactionLogEntry;
import com.iqser.red.service.peristence.v1.server.migration.model.MigrationViewedPage;
import com.iqser.red.service.peristence.v1.server.migration.model.NewIdForDossierTemplate;
import com.iqser.red.service.peristence.v1.server.migration.model.Status;
import com.iqser.red.service.peristence.v1.server.migration.model.VersionRow;
import com.iqser.red.service.peristence.v1.server.migration.model.VersionType;
import com.iqser.red.service.peristence.v1.server.migration.model.*;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.audit.AuditEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DigitalSignatureEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisMappingEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.notification.NotificationEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.service.EncryptionDecryptionService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ColorsRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ForceRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ImageRecategorizationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisChangeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisMappingRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ManualRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RemoveRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RuleSetRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ViewedPagesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.Change;
import com.iqser.red.service.redaction.v1.model.ChangeType;
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import static java.util.stream.Collectors.toList;
@SuppressWarnings("PMD")
@Slf4j
@ -264,41 +179,42 @@ public class MigrationService {
List<FileEntity> files = fileRepository.findAll();
files.stream().filter(file -> file.getHardDeletedTime() == null).forEach(file -> {
migrateRedactionLog(file.getDossierId(), file.getId(), file.getLastProcessed());
migrateRedactionLog(file);
});
}
@SneakyThrows
private void migrateRedactionLog(String dossierId, String fileId, OffsetDateTime lastProcessed) {
private void migrateRedactionLog(FileEntity file) {
try {
byte[] oldRedactionLogBytes = fileManagementStorageService.getStoredObjectBytes(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
byte[] oldRedactionLogBytes = fileManagementStorageService.getStoredObjectBytes(StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG));
if (oldRedactionLogBytes == null) {
log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", dossierId, fileId);
log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", file.getDossierId(), file.getId());
return;
}
var oldRedactionLog = objectMapper.readValue(oldRedactionLogBytes, MigrationRedactionLog.class);
RedactionLog redactionLog = new RedactionLog(oldRedactionLog.getAnalysisVersion(), oldRedactionLog.getRedactionLogEntry()
RedactionLog redactionLog = new RedactionLog(oldRedactionLog.getAnalysisVersion(),
file.getNumberOfAnalyses(), oldRedactionLog.getRedactionLogEntry()
.stream()
.map(r -> convertRedactionLogEntry(r, fileId, lastProcessed))
.map(r -> convertRedactionLogEntry(r, file.getId(), file.getLastProcessed(),file.getNumberOfAnalyses()))
.collect(toList()), oldRedactionLog.getLegalBasis(), oldRedactionLog.getDictionaryVersion(), oldRedactionLog
.getDossierDictionaryVersion(), oldRedactionLog.getRulesVersion(), oldRedactionLog.getLegalBasisVersion());
storageService.storeObject("backup/" + StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), objectMapper
storageService.storeObject("backup/" + StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG), objectMapper
.writeValueAsBytes(oldRedactionLogBytes));
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), objectMapper
storageService.storeObject(StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG), objectMapper
.writeValueAsBytes(redactionLog));
} catch (StorageObjectDoesNotExist e) {
log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", dossierId, fileId);
log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", file.getDossierId(), file.getId());
}
}
private RedactionLogEntry convertRedactionLogEntry(MigrationRedactionLogEntry oldRedactionLogEntry, String fileId,
OffsetDateTime lastProcessed) {
OffsetDateTime lastProcessed, int analysisNumber) {
return RedactionLogEntry.builder()
.id(oldRedactionLogEntry.getId())
@ -335,7 +251,7 @@ public class MigrationService {
.excluded(oldRedactionLogEntry.isExcluded())
.recategorizationType(oldRedactionLogEntry.getRecategorizationType())
.legalBasisChangeValue(oldRedactionLogEntry.getLegalBasisChangeValue())
.changes(List.of(new Change(ChangeType.ADDED, lastProcessed)))
.changes(List.of(new Change(analysisNumber, ChangeType.ADDED, lastProcessed)))
.engines(getEngine(oldRedactionLogEntry))
.reference(null)
.rectangle(oldRedactionLogEntry.isRectangle())
@ -346,7 +262,7 @@ public class MigrationService {
private Set<Engine> getEngine(MigrationRedactionLogEntry oldRedactionLogEntry) {
if (oldRedactionLogEntry.isManual()) {
return new HashSet<Engine>();
return new HashSet<>();
}
if (oldRedactionLogEntry.isDictionaryEntry() || oldRedactionLogEntry.isDossierDictionaryEntry()) {
return Set.of(Engine.DICTIONARY);

7
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -101,7 +101,11 @@ public class FileStatusService {
public void setStatusSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) {
fileStatusPersistenceService.updateWorkflowStatus(fileId, analyzeResult.getNumberOfPages(), analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion());
fileStatusPersistenceService.updateWorkflowStatus(fileId, analyzeResult.getNumberOfPages(),
analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(),
analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(),
analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(),
analyzeResult.getAnalysisNumber());
}
@ -267,6 +271,7 @@ public class FileStatusService {
var analyseRequest = AnalyzeRequest.builder()
.messageType(reanalyse ? MessageType.REANALYSE : MessageType.FULL_ANALYSE)
.dossierId(dossierId)
.analysisNumber(fileStatus.getNumberOfAnalyses()+1)
.sectionsToReanalyse(sectionsToReanalyse)
.fileId(fileId)
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))

3
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java
View File

@ -58,7 +58,8 @@ public class FileTesterAndProvider {
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isGreaterThanOrEqualTo(1);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(new RedactionLog(1, List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()),
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(new RedactionLog(1, 1,
List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()),
null, 0, 0, 0, 0)));
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, objectMapper.writeValueAsBytes(new SectionGrid()));

3
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -129,7 +129,8 @@ public abstract class AbstractPersistenceServerServiceTest {
when(redactionClient.annotate(Mockito.any())).thenReturn(new AnnotateResponse("document".getBytes()));
when(redactionClient.sections(Mockito.any())).thenReturn(new RedactionResult());
when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(new RedactionLog(1, Lists.newArrayList(), null, 0, 0, 0, 0));
when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(new RedactionLog(1, 1,
Lists.newArrayList(), null, 0, 0, 0, 0));
}
@After

2
persistence-service-v1/pom.xml
View File

@ -25,7 +25,7 @@
</modules>
<properties>
<redaction-service.version>3.43.0</redaction-service.version>
<redaction-service.version>3.56.0</redaction-service.version>
<search-service.version>2.10.0</search-service.version>
<pdftron-redaction-service.version>3.17.0</pdftron-redaction-service.version>
<redaction-report-service.version>3.19.0</redaction-report-service.version>