Merge branch 'hotfix-missing-migration' into 'release/2.589.x'

migration hotfixes

See merge request redactmanager/persistence-service!854
This commit is contained in:
Maverick Studer 2024-11-13 11:20:41 +01:00
commit aaa87c7a74
20 changed files with 131 additions and 62 deletions

View File

@ -24,7 +24,7 @@ import lombok.extern.slf4j.Slf4j;
@Setter
@Service
@SuppressWarnings("PMD")
public class ReduceTextFileSizeMigration10 extends Migration {
public class V10ReduceTextFileSizeMigration extends Migration {
private static final String NAME = "Reduce TEXT filesize migration";
private static final long VERSION = 10;
@ -42,7 +42,7 @@ public class ReduceTextFileSizeMigration10 extends Migration {
private FileManagementStorageService fileManagementStorageService;
public ReduceTextFileSizeMigration10() {
public V10ReduceTextFileSizeMigration() {
super(NAME, VERSION);
}

View File

@ -17,7 +17,7 @@ import java.util.List;
@Slf4j
@Setter
@Service
public class MissingFileSizeMigration13 extends Migration {
public class V13MissingFileSizeMigration extends Migration {
private static final String NAME = "Add missing file sizes";
private static final long VERSION = 13;
@ -29,7 +29,7 @@ public class MissingFileSizeMigration13 extends Migration {
private FileManagementStorageService fileManagementStorageService;
public MissingFileSizeMigration13() {
public V13MissingFileSizeMigration() {
super(NAME, VERSION);
}

View File

@ -10,13 +10,13 @@ import org.springframework.stereotype.Service;
@Slf4j
@Setter
@Service
public class FixDossierDictionaryEntryInRedactionLog14 extends Migration {
public class V14FixDossierDictionaryEntryInRedactionLog extends Migration {
private static final String NAME = "Fix dossier dictionary entries in redactionLog for non dossier dictionaries";
private static final long VERSION = 14;
public FixDossierDictionaryEntryInRedactionLog14() {
public V14FixDossierDictionaryEntryInRedactionLog() {
super(NAME, VERSION);
}

View File

@ -28,7 +28,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class ManualRedactionTypeRenameMigration15 extends Migration {
public class V15ManualRedactionTypeRenameMigration extends Migration {
@Autowired
private FileStatusPersistenceService fileStatusPersistenceService;
@ -40,7 +40,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration {
private ManualRedactionService manualRedactionService;
public ManualRedactionTypeRenameMigration15() {
public V15ManualRedactionTypeRenameMigration() {
super(NAME, VERSION);
}

View File

@ -12,13 +12,13 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class RankDeDuplicationMigration16 extends Migration {
public class V16RankDeDuplicationMigration extends Migration {
@Autowired
private RankDeDuplicationService rankDeDuplicationService;
public RankDeDuplicationMigration16() {
public V16RankDeDuplicationMigration() {
super(NAME, VERSION);
}

View File

@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class MigrateImportedRedactionsFiles17 extends Migration {
public class V17MigrateImportedRedactionsFiles extends Migration {
@Autowired
private FileStatusPersistenceService fileStatusPersistenceService;
@ -34,7 +34,7 @@ public class MigrateImportedRedactionsFiles17 extends Migration {
private static final long VERSION = 17;
public MigrateImportedRedactionsFiles17() {
public V17MigrateImportedRedactionsFiles() {
super(NAME, VERSION);
}

View File

@ -11,7 +11,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class StorageToMongoMigration18 extends Migration {
public class V18StorageToMongoMigration extends Migration {
private final StorageToMongoCopyService storageToMongoCopyService;
@ -19,7 +19,7 @@ public class StorageToMongoMigration18 extends Migration {
private static final long VERSION = 18;
public StorageToMongoMigration18(StorageToMongoCopyService storageToMongoCopyService) {
public V18StorageToMongoMigration(StorageToMongoCopyService storageToMongoCopyService) {
super(NAME, VERSION);
this.storageToMongoCopyService = storageToMongoCopyService;

View File

@ -16,7 +16,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class AddGraphicDictionaryType19 extends Migration {
public class V19AddGraphicDictionaryType extends Migration {
@Autowired
private DossierTemplatePersistenceService dossierTemplatePersistenceService;
@ -31,7 +31,7 @@ public class AddGraphicDictionaryType19 extends Migration {
private static final long VERSION = 19;
public AddGraphicDictionaryType19() {
public V19AddGraphicDictionaryType() {
super(NAME, VERSION);
}

View File

@ -16,7 +16,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class DocumineLayoutRewriteMigration20 extends Migration {
public class V20DocumineLayoutRewriteMigration extends Migration {
private static final String NAME = "Reanalyse layout for not approved Documine files";
private static final long VERSION = 20;
@ -36,7 +36,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration {
public DocumineLayoutRewriteMigration20() {
public V20DocumineLayoutRewriteMigration() {
super(NAME, VERSION);
}

View File

@ -6,7 +6,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ApplicationType;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.migration.StorageToMongoCopyService;
@ -19,7 +18,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class ComponentOverridesMigration21 extends Migration {
public class V21ComponentOverridesMigration extends Migration {
private static final String NAME = "Migrate component overrides to mongoDB and create component definitions";
private static final long VERSION = 21;
@ -128,7 +127,7 @@ public class ComponentOverridesMigration21 extends Migration {
StorageToMongoCopyService storageToMongoCopyService;
public ComponentOverridesMigration21() {
public V21ComponentOverridesMigration() {
super(NAME, VERSION);
}

View File

@ -0,0 +1,70 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class V22DocumineHeadlineDetectionMigration extends Migration {
private static final String NAME = "Reanalyse not approved Documine files after new headline detection";
private static final long VERSION = 22;
@Autowired
private FileStatusService fileStatusService;
@Autowired
private DossierPersistenceService dossierPersistenceService;
@Autowired
private FileStatusPersistenceService fileStatusPersistenceService;
@Value("${application.type}")
private String applicationType;
public V22DocumineHeadlineDetectionMigration() {
super(NAME, VERSION);
}
@Override
protected void migrate() {
log.info("Starting migration DocumineHeadlineDetectionMigration22");
if (!applicationType.equalsIgnoreCase("DocuMine")) {
log.info("Skipping DocumineHeadlineDetectionMigration22 as application type is not DocuMine!");
return;
}
var dossiers = dossierPersistenceService.findAllDossiers();
dossiers.forEach(dossier -> {
if (dossier.getHardDeletedTime() == null) {
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId());
log.info("Start migration of dossier {}", dossier.getId());
files.forEach(file -> {
if (file.getHardDeletedTime() == null && !file.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
log.info("Set full reanalyse for file {}", file.getId());
fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true);
log.info("Finished migration of file {}", file.getId());
}
});
log.info("Finished migration of dossier {}", dossier.getId());
}
});
}
}

View File

@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class ManualChangesReorderingMigration23 extends Migration {
public class V23ManualChangesReorderingMigration extends Migration {
private static final String NAME = "Migration for reordering mixed up manual changes";
private static final long VERSION = 23;
@ -29,9 +29,9 @@ public class ManualChangesReorderingMigration23 extends Migration {
private final FileStatusService fileStatusService;
public ManualChangesReorderingMigration23(EntityLogEntryDocumentRepository entityLogEntryDocumentRepository,
AddRedactionPersistenceService addRedactionPersistenceService,
FileStatusService fileStatusService) {
public V23ManualChangesReorderingMigration(EntityLogEntryDocumentRepository entityLogEntryDocumentRepository,
AddRedactionPersistenceService addRedactionPersistenceService,
FileStatusService fileStatusService) {
super(NAME, VERSION);
this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;

View File

@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j;
@Setter
@Service
public class StorageToMongoMigration24 extends Migration {
public class V24StorageToMongoMigration extends Migration {
private final StorageToMongoCopyService storageToMongoCopyService;
private final MongoTemplate mongoTemplate;
@ -29,7 +29,7 @@ public class StorageToMongoMigration24 extends Migration {
private static final long VERSION = 24;
public StorageToMongoMigration24(StorageToMongoCopyService storageToMongoCopyService, MongoTemplate mongoTemplate) {
public V24StorageToMongoMigration(StorageToMongoCopyService storageToMongoCopyService, MongoTemplate mongoTemplate) {
super(NAME, VERSION);
this.storageToMongoCopyService = storageToMongoCopyService;

View File

@ -1,15 +1,9 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
@ -25,7 +19,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class AddDateFormatsToTemplatesMigration25 extends Migration {
public class V25AddDateFormatsToTemplatesMigration extends Migration {
private static final String NAME = "Migration for adding date formats files to dossier templates";
private static final long VERSION = 25;
@ -41,7 +35,7 @@ public class AddDateFormatsToTemplatesMigration25 extends Migration {
private String applicationType;
public AddDateFormatsToTemplatesMigration25() {
public V25AddDateFormatsToTemplatesMigration() {
super(NAME, VERSION);
}

View File

@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class RuleFileUpdateMigration26 extends Migration {
public class V26RuleFileUpdateMigration extends Migration {
private final DossierTemplateRepository dossierTemplateRepository;
private final RulesPersistenceService rulesPersistenceService;
@ -29,7 +29,7 @@ public class RuleFileUpdateMigration26 extends Migration {
private static final long VERSION = 26;
public RuleFileUpdateMigration26(DossierTemplateRepository dossierTemplateRepository, RulesPersistenceService rulesPersistenceService) {
public V26RuleFileUpdateMigration(DossierTemplateRepository dossierTemplateRepository, RulesPersistenceService rulesPersistenceService) {
super(NAME, VERSION);
this.dossierTemplateRepository = dossierTemplateRepository;

View File

@ -14,7 +14,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class QueueRenameMigration27 extends Migration {
public class V27QueueRenameMigration extends Migration {
private final AmqpAdmin amqpAdmin;
@ -86,7 +86,7 @@ public class QueueRenameMigration27 extends Migration {
"visual_layout_parsing_service_response_queue");
public QueueRenameMigration27(AmqpAdmin amqpAdmin) {
public V27QueueRenameMigration(AmqpAdmin amqpAdmin) {
super(NAME, VERSION);
this.amqpAdmin = amqpAdmin;

View File

@ -12,15 +12,15 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class AddTechnicalNameToJustifications22 extends Migration {
public class V28AddTechnicalNameToJustifications extends Migration {
private static final String NAME = "Migration to add a technical name to justifications";
private static final long VERSION = 22;
private static final long VERSION = 28;
@Autowired
private LegalBasisMigrationService legalBasisMigrationService;
public AddTechnicalNameToJustifications22() {
public V28AddTechnicalNameToJustifications() {
super(NAME, VERSION);
}

View File

@ -1,12 +1,11 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.Map;
import org.springframework.stereotype.Service;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisMappingRepository;
import jakarta.transaction.Transactional;
@ -19,14 +18,12 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LegalBasisMigrationService {
private static final HashFunction hashFunction = Hashing.murmur3_128();
private static final Map<String, String> technicalNameMapping = Map.ofEntries(Map.entry("Article 39(e)(3) of Regulation (EC) No 178/2002",
"personal_data_geolocation_article_39e3"),
Map.entry("Article 39(e)(2) of Regulation (EC) No 178/2002",
"vertebrate_study_personal_data_geolocation_article_39e2"),
Map.entry(
"Article 63(2)(a) of Regulation (EC) No 1107/2009 (reference to Article 39 of Regulation EC No 178/2002)",
"Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)",
"manufacturing_production_process"),
Map.entry(
"Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)",
@ -50,15 +47,19 @@ public class LegalBasisMigrationService {
Map.entry("Article 63(2)(c) of Regulation (EC) No 1107/2009", "results_production_batches"),
Map.entry("Article 63(2)(d) of Regulation (EC) No 1107/2009",
"composition_plant_protection_product"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2g)", "names_addresses_persons"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2d)", "methods_analysis_impurities"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2a)", "method_manufacture"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2b)", "specification_impurity_active_substance"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2c)", "results_production_batches"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2d)", "methods_analysis_impurities"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2e)", "links_producer_applicant"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2f)", "composition_plant_protection_product"),
Map.entry("Reg (EC) No 1107/2009 Art. 63 (2g)", "names_addresses_persons"),
Map.entry("Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)",
"personal_data_geolocation"),
Map.entry("Article 4(2), first indent, Regulation (EC) No 1049/2001 (commercial interest)",
"proprietary_information"),
Map.entry("3. n/a", "n_a"));
Map.entry("N/A", "n_a"),
Map.entry("n-a.", "n-a."));
private final LegalBasisMappingRepository legalBasisMappingRepository;
@ -70,17 +71,22 @@ public class LegalBasisMigrationService {
this.legalBasisMappingRepository.findAll()
.stream()
.peek(entry -> entry.getLegalBasis()
.forEach(lb -> {
lb.setTechnicalName(getTechnicalNameOrHash(lb.getReason()));
}))
.forEach(lb -> lb.setTechnicalName(getTechnicalNameOrFallback(lb))))
.forEach(legalBasisMappingRepository::save);
log.info("Finishing migration: Adding technical names to legal basis");
}
private static String getTechnicalNameOrHash(String reason) {
private static String getTechnicalNameOrFallback(LegalBasisEntity legalBasisEntity) {
return technicalNameMapping.getOrDefault(reason, String.valueOf(hashFunction.hashString(reason, StandardCharsets.UTF_8)));
return technicalNameMapping.getOrDefault(legalBasisEntity.getReason(), generateNewTechnicalName(legalBasisEntity.getName()));
}
private static String generateNewTechnicalName(String name) {
return name.toLowerCase(Locale.ROOT).replaceAll("\\.", "").replaceAll("\\s+", "_");
}
}

View File

@ -8,7 +8,7 @@ import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.ReduceTextFileSizeMigration10;
import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.V10ReduceTextFileSizeMigration;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileUtils;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -19,7 +19,7 @@ import lombok.SneakyThrows;
public class ReduceFileSizeMigrationTest extends AbstractPersistenceServerServiceTest {
@Autowired
private ReduceTextFileSizeMigration10 reduceTextFileSizeMigration10;
private V10ReduceTextFileSizeMigration v10ReduceTextFileSizeMigration;
@Autowired
private ObjectMapper objectMapper;
@ -33,7 +33,7 @@ public class ReduceFileSizeMigrationTest extends AbstractPersistenceServerServic
storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId("dossierId", "fileId", FileType.TEXT), responseJson.getInputStream());
reduceTextFileSizeMigration10.migrateFile("dossierId", "fileId");
v10ReduceTextFileSizeMigration.migrateFile("dossierId", "fileId");
// readJSONObject will not work here.
var text = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId("dossierId", "fileId", FileType.TEXT));

View File

@ -20,19 +20,19 @@ import org.springframework.test.context.TestPropertySource;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateImportService;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DateFormatsEntity;
import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.AddDateFormatsToTemplatesMigration25;
import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.V25AddDateFormatsToTemplatesMigration;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import lombok.SneakyThrows;
@TestPropertySource(properties = {"application.type=DocuMine"})
public class AddDateFormatsToTemplatesMigration25Test extends AbstractPersistenceServerServiceTest {
public class V25AddDateFormatsToTemplatesMigrationTest extends AbstractPersistenceServerServiceTest {
private static final String USER_ID = "TestUserId";
@Autowired
private AddDateFormatsToTemplatesMigration25 migration;
private V25AddDateFormatsToTemplatesMigration migration;
@Autowired
private DossierTemplateImportService dossierTemplateImportService;