Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #252: Removed 2.5.x migration for 3.1

Browse Source 
Merge in RED/persistence-service from migrationFix to master

* commit '553ee374792c740f93137738473b9113d5e69b9c':
  Removed 2.5.x migration for 3.1
This commit is contained in:
Dominique Eiflaender 2022-02-21 17:02:48 +01:00
commit f6efa1a4de
33 changed files with 5 additions and 2386 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationEncryptionDecryptionService.java
View File

@ -1,49 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.Arrays;
import java.util.Base64;
import javax.annotation.PostConstruct;
import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import lombok.SneakyThrows;
@Service
public class MigrationEncryptionDecryptionService {
@Value("${configuration-service.crypto.key:redaction}")
private String key;
private SecretKeySpec secretKey;
@SneakyThrows
@PostConstruct
protected void postConstruct() {
byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
var sha = MessageDigest.getInstance("SHA-1");
keyBytes = sha.digest(keyBytes);
keyBytes = Arrays.copyOf(keyBytes, 16);
secretKey = new SecretKeySpec(keyBytes, "AES");
}
@SneakyThrows
public String encrypt(String strToEncrypt) {
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
return Base64.getEncoder().encodeToString(cipher.doFinal(strToEncrypt.getBytes(StandardCharsets.UTF_8)));
}
@SneakyThrows
public String decrypt(String strToDecrypt) {
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING");
cipher.init(Cipher.DECRYPT_MODE, secretKey);
return new String(cipher.doFinal(Base64.getDecoder().decode(strToDecrypt)), StandardCharsets.UTF_8);
}
}

1137
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java
View File

File diff suppressed because it is too large Load Diff

35
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java
View File

@ -1,35 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationContext;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class MigrationStarterService {
// This is a seperate class because exit 0 causes rollback if the method is @Transactional.
private final MigrationService migrationService;
private final FileManagementServiceSettings settings;
private final ApplicationContext ctx;
@EventListener(ApplicationReadyEvent.class)
public void runMigration() {
if (settings.isMigrateOnly()) {
migrationService.migrate();
System.exit(SpringApplication.exit(ctx, () -> 0));
}
}
}

7
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/image/MigrateImageService.java
View File

@ -14,6 +14,8 @@ import org.springframework.stereotype.Service;
import java.util.List;
import javax.transaction.Transactional;
@Slf4j
@Service
@RequiredArgsConstructor
@ -25,6 +27,7 @@ public class MigrateImageService {
@SneakyThrows
@Transactional
public void migrateImagesFiles() {
List<FileEntity> files = fileRepository.findAll();
@ -50,7 +53,7 @@ public class MigrateImageService {
ImageMetadata imageMetadata = new ImageMetadata();
Classification classification = new Classification();
classification.setLabel(entry.getType());
classification.setLabel(entry.getType().equals("ocr") || entry.getType().equals("image") ? "other" : entry.getType());
imageMetadata.setClassification(classification);
Position position = new Position();
@ -65,7 +68,7 @@ public class MigrateImageService {
imageMetadata.setGeometry(geometry);
Filters filters = new Filters();
filters.setAllPassed(!entry.getType().equals("image"));
filters.setAllPassed(!entry.getType().equals("ocr") && !entry.getType().equals("image"));
imageMetadata.setFilters(filters);
imageServiceResponse.getImageMetadata().add(imageMetadata);

24
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/CommentRow.java
View File

@ -1,24 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CommentRow {
private String id;
private OffsetDateTime date;
private String text;
private String user;
private OffsetDateTime softDeletedTime;
private String fileId;
private String annotationId;
}

20
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/ConfigurationRow.java
View File

@ -1,20 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.io.Serializable;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ConfigurationRow implements Serializable {
private String dossierTemplateId;
public String key;
public String value;
}

22
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/DigitalSignatureModel.java
View File

@ -1,22 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor(access = AccessLevel.PRIVATE)
@AllArgsConstructor
@Builder
public class DigitalSignatureModel {
private String location;
private String reason;
private String contactInfo;
private String certificateName;
private byte[] privateKey;
private String password;
}

19
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/DossierAttributeRow.java
View File

@ -1,19 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class DossierAttributeRow {
private String dossierAttributeId;
private String value;
private String dossierId;
}

19
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/DossierAttributesConfig.java
View File

@ -1,19 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class DossierAttributesConfig {
private List<DossierAttributeConfigEntity> dossierAttributeConfigs = new ArrayList<>();
}

23
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/EntryRow.java
View File

@ -1,23 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.io.Serializable;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class EntryRow implements Serializable {
private String dossierTemplateId;
private String dossierId;
private String type;
private String value;
private long version;
private boolean deleted;
}

21
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/FileAttributesConfig.java
View File

@ -1,21 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class FileAttributesConfig {
private String filenameMappingColumnHeaderName;
private String delimiter;
private List<FileAttributeConfigEntity> fileAttributeConfigs = new ArrayList<>();
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/IdRemovalRow.java
View File

@ -1,27 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class IdRemovalRow {
private String id;
private String user;
private AnnotationStatus status;
private boolean removeFromDictionary;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
private String fileId;
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/ManualForceRedactionRow.java
View File

@ -1,27 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualForceRedactionRow {
private String id;
private String user;
private AnnotationStatus status;
private String legalBasis;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
private String fileId;
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/ManualImageRecategorizationRow.java
View File

@ -1,27 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualImageRecategorizationRow {
private String id;
private String user;
private AnnotationStatus status;
private String type;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
private String fileId;
}

29
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/ManualLegalBasisChangeRow.java
View File

@ -1,29 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualLegalBasisChangeRow {
private String id;
private String user;
private AnnotationStatus status;
private String legalBasis;
private String section;
private String value;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
private String fileId;
}

42
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/ManualRedactionEntryRow.java
View File

@ -1,42 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualRedactionEntryRow {
private String id;
private String user;
private String type;
private String value;
private String reason;
private String legalBasis;
private String section;
private boolean rectangle;
@Builder.Default
private List<Rectangle> positions = new ArrayList<>();
private AnnotationStatus status;
private boolean addToDictionary;
private boolean addToDossierDictionary;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
private String fileId;
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationAuditModel.java
View File

@ -1,27 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class MigrationAuditModel {
private String recordId;
private OffsetDateTime recordDate;
private String objectId;
private String category;
private String userId;
private String message;
private Map<String, Object> details;
}

21
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationComment.java
View File

@ -1,21 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class MigrationComment {
private String id;
private OffsetDateTime date;
private String text;
private String user;
}

59
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationDossier.java
View File

@ -1,59 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import java.util.HashSet;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import lombok.Builder;
import lombok.Data;
import lombok.NonNull;
@Data
@Builder
public class MigrationDossier {
private String dossierId;
private String dossierTemplateId;
@NonNull
private String dossierName;
private OffsetDateTime date;
private String description;
private DossierStatus status;
private String ownerId;
@Builder.Default
private Set<String> memberIds = new HashSet<>();
@Builder.Default
private Set<String> approverIds = new HashSet<>();
@Builder.Default
private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
@Builder.Default
private Set<ReportType> reportTypes = new HashSet<>();
private OffsetDateTime dueDate;
@Builder.Default
private Set<String> reportTemplateIds = new HashSet<>();
private boolean watermarkEnabled;
private boolean watermarkPreviewEnabled;
private OffsetDateTime softDeletedTime;
private OffsetDateTime hardDeletedTime;
}

43
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationDossierTemplate.java
View File

@ -1,43 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import java.util.HashSet;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class MigrationDossierTemplate {
private String dossierTemplateId;
private String name;
private String description;
private OffsetDateTime dateAdded;
private OffsetDateTime dateModified;
private String createdBy;
private String modifiedBy;
private OffsetDateTime validFrom;
private OffsetDateTime validTo;
@Builder.Default
private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
private boolean deleted;
}

35
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationDownloadDetails.java
View File

@ -1,35 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class MigrationDownloadDetails {
@Builder.Default
private List<String> fileIds = new ArrayList<>();
@Builder.Default
private Set<ReportType> reportTypes = new HashSet<>();
@Builder.Default
private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
@Builder.Default
private Set<String> reportTemplateIds = new HashSet<>();
}

30
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationDownloadStatus.java
View File

@ -1,30 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class MigrationDownloadStatus {
private String userId;
private String storageId;
private String dossierId;
private String filename;
private String mimeType;
private DownloadStatusValue status;
private OffsetDateTime creationDate;
private OffsetDateTime lastDownload;
private MigrationDownloadDetails downloadDetails;
private long fileSize;
}

16
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationFileAttributes.java
View File

@ -1,16 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.util.HashMap;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class MigrationFileAttributes {
Map<String, String> attributeIdToValue = new HashMap<>();
}

88
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationFileStatus.java
View File

@ -1,88 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import java.util.Set;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class MigrationFileStatus {
private String dossierId;
private String fileId;
private String filename;
private Status status;
private Status lastSuccessfulStatus;
private int numberOfPages;
private OffsetDateTime added;
private OffsetDateTime lastUpdated;
private OffsetDateTime deleted;
private OffsetDateTime lastProcessed;
private int numberOfAnalyses;
private String currentReviewer;
private String lastReviewer;
private OffsetDateTime lastManualRedaction;
private boolean hasRedactions;
private boolean hasHints;
private boolean hasSuggestions;
private boolean hasImages;
private boolean hasUpdates;
private String uploader;
private long dictionaryVersion;
private long rulesVersion;
private long dossierDictionaryVersion;
private long legalBasisVersion;
private OffsetDateTime approvalDate;
private OffsetDateTime lastUploaded;
private long analysisDuration;
private MigrationFileAttributes fileAttributes;
private OffsetDateTime lastOCRTime;
private boolean hasAnnotationComments;
private boolean excluded;
private Set<Integer> excludedPages;
private OffsetDateTime hardDeletedTime;
private OffsetDateTime lastFileAttributeChange;
private int analysisVersion;
}

32
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationNotification.java
View File

@ -1,32 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class MigrationNotification {
private String userId;
private String notificationId;
private String notificationType;
private String issuerId;
@Builder.Default
private Map<String, Object> target = new HashMap<>();
private OffsetDateTime creationDate;
private OffsetDateTime seenDate;
private OffsetDateTime readDate;
private OffsetDateTime softDeleted;
}

28
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationRedactionLog.java
View File

@ -1,28 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class MigrationRedactionLog {
/**
* Version 0 Redaction Logs have manual redactions merged inside them
* Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime.
*/
private long analysisVersion;
private List<MigrationRedactionLogEntry> redactionLogEntry;
private List<LegalBasis> legalBasis;
private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1;
}

64
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationRedactionLogEntry.java
View File

@ -1,64 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode
public class MigrationRedactionLogEntry {
private String id;
private String type;
private String value;
private String reason;
private int matchedRule;
private boolean rectangle;
private String legalBasis;
private boolean redacted;
private boolean isHint;
private boolean isRecommendation;
private String section;
private float[] color;
@Builder.Default
private List<Rectangle> positions = new ArrayList<>();
private int sectionNumber;
private boolean manual;
private AnnotationStatus status;
private ManualRedactionType manualRedactionType;
private String manualRedactionUserId;
private boolean isDictionaryEntry;
private String textBefore;
private String textAfter;
@Builder.Default
private List<MigrationComment> comments = new ArrayList<>();
private int startOffset;
private int endOffset;
private boolean isImage;
private boolean imageHasTransparency;
private boolean isDossierDictionaryEntry;
private boolean excluded;
private String recategorizationType;
private String legalBasisChangeValue;
}

19
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/MigrationViewedPage.java
View File

@ -1,19 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class MigrationViewedPage {
private String fileId;
private String userId;
private int page;
}

13
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/NewIdForDossierTemplate.java
View File

@ -1,13 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class NewIdForDossierTemplate {
private String newId;
private String dossierTemplateId;
}

5
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/Status.java
View File

@ -1,5 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
public enum Status {
UNPROCESSED, REPROCESS, PROCESSING, ERROR, DELETED, UNASSIGNED, UNDER_REVIEW, UNDER_APPROVAL, APPROVED, FULLREPROCESS, OCR_PROCESSING, INDEXING;
}

20
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/VersionRow.java
View File

@ -1,20 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import java.io.Serializable;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class VersionRow implements Serializable {
private String dossierTemplateId;
private String dossierId;
private int versionType;
private long version;
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/model/VersionType.java
View File

@ -1,27 +0,0 @@
package com.iqser.red.service.peristence.v1.server.migration.model;
import lombok.Getter;
public enum VersionType {
RULE(0), DICTIONARY(1), LEGAL_BASIS(2);
@Getter
private int identifier;
VersionType(int identifier){
this.identifier = identifier;
}
public static VersionType forIdentifier(int identifier){
switch(identifier){
case 0:
return RULE;
case 1:
return DICTIONARY;
case 2:
return LEGAL_BASIS;
}
return null;
}
}

336
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/MigrationTest.java
View File

@ -1,336 +0,0 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.integration.client.AuditClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DigitalSignatureClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierAttributeClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierAttributeConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.LegalBasisClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
import com.iqser.red.service.peristence.v1.server.integration.client.NotificationClient;
import com.iqser.red.service.peristence.v1.server.integration.client.RedactionLogClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ReportTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.client.RulesClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ViewedPagesClient;
import com.iqser.red.service.peristence.v1.server.integration.client.WatermarkClient;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.peristence.v1.server.migration.MigrationService;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.model.audit.AuditSearchRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import lombok.SneakyThrows;
@Ignore
public class MigrationTest extends AbstractPersistenceServerServiceTest {
@Autowired
private MigrationService migrationService;
@MockBean
private FileManagementStorageService fileManagementStorageService;
@Autowired
private DossierTemplateClient dossierTemplateClient;
@Autowired
private ReportTemplateClient reportTemplateClient;
@Autowired
private DossierAttributeConfigClient dossierAttributeConfigClient;
@Autowired
private FileAttributeConfigClient fileAttributeConfigClient;
@Autowired
private LegalBasisClient legalBasisClient;
@Autowired
private WatermarkClient watermarkClient;
@Autowired
private DictionaryClient dictionaryClient;
@Autowired
private DossierClient dossierClient;
@Autowired
private FileClient fileClient;
@Autowired
private AuditClient auditClient;
@Autowired
private NotificationClient notificationClient;
@Autowired
private DownloadClient downloadClient;
@Autowired
private RulesClient rulesClient;
@Autowired
private ViewedPagesClient viewedPagesClient;
@Autowired
private DossierAttributeClient dossierAttributeClient;
@Autowired
private ManualRedactionClient manualRedactionClient;
@Autowired
private RedactionLogClient redactionLogClient;
@Autowired
private DigitalSignatureClient digitalSignatureClient;
@Test
@SneakyThrows
public void testMigration() {
ClassPathResource dossierTemplatesJson = new ClassPathResource("files/migration/dossier_template_v2.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("dossier_template_v2"))).thenReturn(dossierTemplatesJson
.getInputStream()
.readAllBytes());
ClassPathResource reportTemplatesJson = new ClassPathResource("files/migration/report_template_table_v4.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("report_template_table_v4"))).thenReturn(reportTemplatesJson
.getInputStream()
.readAllBytes());
ClassPathResource configurationJson = new ClassPathResource("files/migration/configuration_table_v3.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("configuration_table_v3"))).thenReturn(configurationJson
.getInputStream()
.readAllBytes());
ClassPathResource typesJson = new ClassPathResource("files/migration/dictionary_table_v6.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("dictionary_table_v6"))).thenReturn(typesJson
.getInputStream()
.readAllBytes());
ClassPathResource entriesJson = new ClassPathResource("files/migration/entry_table_v6.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("entry_table_v6"))).thenReturn(entriesJson
.getInputStream()
.readAllBytes());
ClassPathResource dossiersJson = new ClassPathResource("files/migration/dossier_v4.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("dossier_v4"))).thenReturn(dossiersJson.getInputStream()
.readAllBytes());
ClassPathResource fileStatusJson = new ClassPathResource("files/migration/file_status_v13.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("file_status_v13"))).thenReturn(fileStatusJson
.getInputStream()
.readAllBytes());
ClassPathResource auditJson = new ClassPathResource("files/migration/audit_record_v2.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("audit_record_v2"))).thenReturn(auditJson.getInputStream()
.readAllBytes());
ClassPathResource notificationJson = new ClassPathResource("files/migration/notification_v1.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("notification_v1"))).thenReturn(notificationJson
.getInputStream()
.readAllBytes());
ClassPathResource downloadJson = new ClassPathResource("files/migration/download_status_v3.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("download_status_v3"))).thenReturn(downloadJson
.getInputStream()
.readAllBytes());
ClassPathResource versionsJson = new ClassPathResource("files/migration/version_table_v4.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("version_table_v4"))).thenReturn(versionsJson
.getInputStream()
.readAllBytes());
ClassPathResource viewedPagesJson = new ClassPathResource("files/migration/viewed_pages_v2.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("viewed_pages_v2"))).thenReturn(viewedPagesJson
.getInputStream()
.readAllBytes());
ClassPathResource dossierAttributesJson = new ClassPathResource("files/migration/dossier_attributes_v1.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("dossier_attributes_v1"))).thenReturn(dossierAttributesJson
.getInputStream()
.readAllBytes());
ClassPathResource manualAddJson = new ClassPathResource("files/migration/manual_redaction_annotations_v5.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("manual_redaction_annotations_v5"))).thenReturn(manualAddJson
.getInputStream()
.readAllBytes());
ClassPathResource forceJson = new ClassPathResource("files/migration/manual_force_redaction_v3.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("manual_force_redaction_v3"))).thenReturn(forceJson
.getInputStream()
.readAllBytes());
ClassPathResource imageRecategorizationJson = new ClassPathResource("files/migration/manual_image_recategorization_v3.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("manual_image_recategorization_v3"))).thenReturn(imageRecategorizationJson
.getInputStream()
.readAllBytes());
ClassPathResource manualLegalBasisChangeJson = new ClassPathResource("files/migration/manual_legal_basis_change_v3.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("manual_legal_basis_change_v3"))).thenReturn(manualLegalBasisChangeJson
.getInputStream()
.readAllBytes());
ClassPathResource removeRedactionsJson = new ClassPathResource("files/migration/manual_redaction_remove_v2.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("manual_redaction_remove_v2"))).thenReturn(removeRedactionsJson
.getInputStream()
.readAllBytes());
ClassPathResource commentsJson = new ClassPathResource("files/migration/manual_redaction_comments_v3.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("manual_redaction_comments_v3"))).thenReturn(commentsJson
.getInputStream()
.readAllBytes());
ClassPathResource confTable1Json = new ClassPathResource("files/migration/configuration_table_v1.json");
when(fileManagementStorageService.getStoredObjectBytes(getMigrationId("configuration_table_v1"))).thenReturn(confTable1Json
.getInputStream()
.readAllBytes());
when(fileManagementStorageService.getStoredObjectBytes(String.format("rules/%s-rules.drl", "6d945ebb-604b-4781-a72b-0f07e8ceb3e6")))
.thenReturn("Rules".getBytes(StandardCharsets.UTF_8));
when(fileManagementStorageService.getStoredObjectBytes(String.format("rules/%s-rules.drl", "49156a1f-30cc-412f-8778-eb5bd842d709")))
.thenReturn("Rules".getBytes(StandardCharsets.UTF_8));
when(fileManagementStorageService.getStoredObjectBytes(String.format("rules/%s-rules.drl", "9b7bd575-4566-4408-984f-b26da1d2616e")))
.thenReturn("Rules".getBytes(StandardCharsets.UTF_8));
ClassPathResource redactionLog = new ClassPathResource("files/migration/220ab22e443c8e32d3778b0937b03170.REDACTION_LOG.json");
when(fileManagementStorageService.getStoredObjectBytes(StorageIdUtils.getStorageId("b2c9c4e5-5382-4862-9402-5b619c41a68c", "220ab22e443c8e32d3778b0937b03170", FileType.REDACTION_LOG)))
.thenReturn(redactionLog.getInputStream().readAllBytes());
migrationService.migrate();
var allTemplates = dossierTemplateClient.getAllDossierTemplates();
assertThat(allTemplates.size()).isEqualTo(2);
allTemplates.forEach(dossierTemplate -> {
var reportTemplates = reportTemplateClient.getAvailableReportTemplates(dossierTemplate.getId());
assertThat(reportTemplates.size()).isEqualTo(4);
var dossierAttributesConfig = dossierAttributeConfigClient.getDossierAttributes(dossierTemplate.getId());
if (dossierTemplate.getId().equals("9b7bd575-4566-4408-984f-b26da1d2616e")) {
assertThat(dossierAttributesConfig.size()).isEqualTo(1);
}
var fileAtrributesConfig = fileAttributeConfigClient.getFileAttributeConfigs(dossierTemplate.getId());
assertThat(fileAtrributesConfig.size()).isEqualTo(5);
var legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplate.getId());
if (dossierTemplate.getId().equals("9b7bd575-4566-4408-984f-b26da1d2616e")) {
assertThat(legalBasis.size()).isEqualTo(7);
} else {
assertThat(legalBasis.size()).isEqualTo(9);
}
var legalBasisVersion = legalBasisClient.getVersion(dossierTemplate.getId());
assertThat(legalBasisVersion).isEqualTo(1);
var watermark = watermarkClient.getWatermark(dossierTemplate.getId());
assertThat(watermark).isNotNull();
var colors = dictionaryClient.getColors(dossierTemplate.getId());
assertThat(colors).isNotNull();
var types = dictionaryClient.getAllTypesForDossierTemplate(dossierTemplate.getId());
types.forEach(type -> {
var entries = dictionaryClient.getEntriesForType(type.getTypeId());
if (dossierTemplate.getId().equals("49156a1f-30cc-412f-8778-eb5bd842d709") && type.getType()
.equals("CBI_address")) {
assertThat(entries.size()).isEqualTo(30);
}
});
var dictionaryVersion = dictionaryClient.getVersion(dossierTemplate.getId());
assertThat(dictionaryVersion).isGreaterThan(0);
if (dossierTemplate.getId().equals("6d945ebb-604b-4781-a72b-0f07e8ceb3e6")) {
assertThat(types.size()).isEqualTo(6);
} else if (dossierTemplate.getId().equals("9b7bd575-4566-4408-984f-b26da1d2616e")) {
assertThat(types.size()).isEqualTo(19);
} else {
assertThat(types.size()).isEqualTo(14);
}
var rules = rulesClient.getRules(dossierTemplate.getId());
assertThat(rules.getValue()).isEqualTo("Rules");
var rulesVersion = rulesClient.getVersion(dossierTemplate.getId());
assertThat(rulesVersion).isGreaterThan(0);
});
var dossiers = dossierClient.getAllDossiers(false, false);
assertThat(dossiers.size()).isEqualTo(3);
dossiers.forEach(dossier -> {
var dossierTypes = dictionaryClient.getAllTypesForDossier(dossier.getId());
assertThat(dossierTypes.size()).isEqualTo(1);
var dossierAttributes = dossierAttributeClient.getDossierAttributes(dossier.getId());
if (dossier.getId().equals("d46267c1-bb19-44ca-8c0c-60fb5d5309c5")) {
assertThat(dossierAttributes.size()).isEqualTo(1);
}
});
List<FileModel> allFiles = new ArrayList<>();
dossiers.forEach(dossier ->{
var files = fileClient.getDossierStatus(dossier.getId());
files.forEach(file -> {
var manualRedactions = manualRedactionClient.getManualRedactions(file.getDossierId(), file.getId());
if (file.getId().equals("a7985c094cac62118c9b350461902f95")) {
assertThat(manualRedactions.getEntriesToAdd().size()).isEqualTo(1);
assertThat(manualRedactions.getForceRedactions().size()).isEqualTo(1);
assertThat(manualRedactions.getImageRecategorization().size()).isEqualTo(0);
assertThat(manualRedactions.getLegalBasisChanges().size()).isEqualTo(1);
assertThat(manualRedactions.getIdsToRemove().size()).isEqualTo(1);
assertThat(manualRedactions.getComments().size()).isEqualTo(1);
}
});
allFiles.addAll(files);
});
assertThat(allFiles.size()).isEqualTo(33);
var digitalSignature = digitalSignatureClient.getDigitalSignature();
assertThat(digitalSignature.getCertificateName()).isNotEmpty();
var auditCategories = auditClient.getCategories();
assertThat(auditCategories.size()).isEqualTo(7);
var auditRecords = auditClient.search(AuditSearchRequest.builder().build());
assertThat(auditRecords.getTotalHits()).isEqualTo(128);
var notifications = notificationClient.getNotifications("abc", true);
assertThat(notifications).isEmpty();
var downloads = downloadClient.getDownloadStatus("f8960240-9973-42e9-9b0f-ed096ff5a018");
assertThat(downloads.size()).isEqualTo(0);
var viewedPages = viewedPagesClient.getViewedPages("a7985c094cac62118c9b350461902f95", "f8960240-9973-42e9-9b0f-ed096ff5a018");
assertThat(viewedPages.size()).isEqualTo(3);
}
private String getMigrationId(String tableName) {
return "migration/" + tableName + ".json";
}
}