diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java index e743db082..9df0b5245 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java @@ -7,23 +7,17 @@ import java.io.BufferedInputStream; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.InputStreamResource; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.context.request.RequestAttributes; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; @@ -161,7 +155,7 @@ public class DownloadController implements DownloadResource { } // otherwise consider the files from dossier var validFilesAndNotProcessed = validFiles.stream() - .filter(f -> !(f.getAnalysisVersion() > 0 && f.getNumberOfAnalyses() > 0)) + .filter(f -> !(f.getAnalysisVersion() > 0 && f.getNumberOfAnalyses() > 0 && !f.isSoftOrHardDeleted())) .collect(Collectors.toList()); if (!validFilesAndNotProcessed.isEmpty()) { throw new BadRequestException("At least a file is in its initial analysis process"); diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java index 29acf7654..a3fb5c1cb 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java @@ -226,7 +226,6 @@ public interface DictionaryResource { @RequestParam(value = "addToDictionary") boolean addToDictionary); - @ResponseStatus(HttpStatus.ACCEPTED) @PostMapping(value = DICTIONARY_REST_PATH + DIFFERENCE + DOSSIER_TEMPLATE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Returns the difference between the dictionaries of the dossier template and all the dossiers inside the template for a list of given types.", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Successfully returned DictionaryDifferenceResponse."), @ApiResponse(responseCode = "400", description = "The request is not valid.")}) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/LegalBasisMappingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/LegalBasisMappingService.java new file mode 100644 index 000000000..3675d1654 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/LegalBasisMappingService.java @@ -0,0 +1,148 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; + +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class LegalBasisMappingService { + + private final HashFunction hashFunction; + private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + + + public LegalBasisMappingService(LegalBasisMappingPersistenceService legalBasisMappingPersistenceService) { + + this.hashFunction = Hashing.murmur3_128(); + this.legalBasisMappingPersistenceService = legalBasisMappingPersistenceService; + } + + + public TechnicalNameResult processLegalBasisAndEntityLogLegalBasisList(String currentLegalBasis, + Map reasonToTechnicalNameMap, + List legalBasisList) { + + TechnicalNameResult result = computeTechnicalName(currentLegalBasis, reasonToTechnicalNameMap); + + if (result == null) { + return null; + } + + addToLegalBasisList(result.technicalName(), currentLegalBasis, legalBasisList); + + return result; + } + + + public TechnicalNameResult processLegalBasis(String currentLegalBasis, Map reasonToTechnicalNameMap) { + + return computeTechnicalName(currentLegalBasis, reasonToTechnicalNameMap); + } + + + private TechnicalNameResult computeTechnicalName(String currentLegalBasis, Map reasonToTechnicalNameMap) { + + if (currentLegalBasis == null || currentLegalBasis.isEmpty()) { + return null; + } + + if (reasonToTechnicalNameMap.containsValue(currentLegalBasis)) { + return new TechnicalNameResult(currentLegalBasis, false); + } + + if (reasonToTechnicalNameMap.containsKey(currentLegalBasis)) { + String technicalName = reasonToTechnicalNameMap.get(currentLegalBasis); + return new TechnicalNameResult(technicalName, false); + } + + String technicalName = hashFunction.hashString(currentLegalBasis, StandardCharsets.UTF_8).toString(); + + return new TechnicalNameResult(technicalName, true); + } + + + public String processTextAndEntityLogLegalBasisList(String text, Map reasonToTechnicalNameMap, List legalBasisList) { + + if (text == null || text.isEmpty()) { + return text; + } + + String updatedValue = text; + boolean updated = false; + + for (Map.Entry entry : reasonToTechnicalNameMap.entrySet()) { + String reason = entry.getKey(); + String technicalName = entry.getValue(); + + if (updatedValue.contains(reason)) { + updatedValue = updatedValue.replace(reason, technicalName); + addToLegalBasisList(technicalName, reason, legalBasisList); + updated = true; + } + } + + if (!updated) { + String technicalName = hashFunction.hashString(text, StandardCharsets.UTF_8).toString(); + updatedValue = technicalName; + addToLegalBasisList(technicalName, text, legalBasisList); + } + + return updatedValue; + } + + + private void addToLegalBasisList(String technicalName, String reason, List legalBasisList) { + + boolean exists = legalBasisList.stream() + .anyMatch(lb -> lb.getTechnicalName().equals(technicalName)); + + if (!exists) { + EntityLogLegalBasis newLegalBasis = createEntityLogLegalBasis(technicalName, reason); + legalBasisList.add(newLegalBasis); + } + } + + + private EntityLogLegalBasis createEntityLogLegalBasis(String technicalName, String reason) { + + EntityLogLegalBasis newLegalBasis = new EntityLogLegalBasis(); + newLegalBasis.setTechnicalName(technicalName); + newLegalBasis.setReason(reason); + newLegalBasis.setName("Generated Legal Basis"); + newLegalBasis.setDescription(""); + return newLegalBasis; + } + + + public Map getReasonToTechnicalNameMap(String dossierTemplateId) { + + List legalBasisMappings = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId); + + if (legalBasisMappings == null) { + legalBasisMappings = new ArrayList<>(); + } + + Map reasonToTechnicalNameMap = new HashMap<>(); + for (LegalBasisEntity lbEntity : legalBasisMappings) { + reasonToTechnicalNameMap.put(lbEntity.getReason(), lbEntity.getTechnicalName()); + } + return reasonToTechnicalNameMap; + + } + + + public record TechnicalNameResult(String technicalName, boolean hashed) { + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/liquibase/TechnicalNameChange.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/liquibase/TechnicalNameChange.java new file mode 100644 index 000000000..72e775b51 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/liquibase/TechnicalNameChange.java @@ -0,0 +1,259 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration.liquibase; + +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; + +import liquibase.change.custom.CustomTaskChange; +import liquibase.database.Database; +import liquibase.database.jvm.JdbcConnection; +import liquibase.exception.CustomChangeException; +import liquibase.exception.ValidationErrors; +import liquibase.resource.ResourceAccessor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@SuppressWarnings("PMD.CloseResource") // we do not want to close the underlying database connection ;) +public class TechnicalNameChange implements CustomTaskChange { + + // there is a bug in our liquibase version which causes custom task changes to be run twice + // that bug got fixed in liquibase 4.25.1 + // when we upgrade the dependency this can then be removed + @Setter + private static boolean skipExecution; + + + @Override + public void execute(Database database) throws CustomChangeException { + + if (skipExecution) { + return; + } + + Connection connection; + + try { + connection = ((JdbcConnection) database.getConnection()).getUnderlyingConnection(); + + String[] tables = {"manual_legal_basis_change", "manual_force_redaction", "manual_recategorization", "manual_redaction"}; + + HashFunction hashFunction = Hashing.murmur3_128(); + + Set allFileIds = collectAllFileIds(connection, tables); + + if (allFileIds.isEmpty()) { + log.info("No rows updated because no files with manual redactions are present."); + return; + } + + Map fileIdToDossierTemplateIdMap = buildFileIdToDossierTemplateIdMap(connection, allFileIds); + + Map reasonDossierTemplateIdToTechnicalNameMap = buildReasonDossierTemplateIdToTechnicalNameMap(connection); + + int totalRowsUpdatedWithTechnicalName = 0; + int totalRowsHashed = 0; + + for (String tableName : tables) { + int[] counts = updateLegalBasisForTable(connection, tableName, hashFunction, fileIdToDossierTemplateIdMap, reasonDossierTemplateIdToTechnicalNameMap); + int rowsUpdatedWithTechnicalName = counts[0]; + int rowsHashed = counts[1]; + + log.info("Table '{}': {} rows updated with technical_name, {} rows hashed.", tableName, rowsUpdatedWithTechnicalName, rowsHashed); + + totalRowsUpdatedWithTechnicalName += rowsUpdatedWithTechnicalName; + totalRowsHashed += rowsHashed; + } + + log.info("Total rows updated with technical_name: {}", totalRowsUpdatedWithTechnicalName); + log.info("Total rows hashed: {}", totalRowsHashed); + + } catch (Exception e) { + throw new CustomChangeException("Error applying technical name change", e); + } + skipExecution = true; + } + + + private Set collectAllFileIds(Connection connection, String[] tables) throws SQLException { + + Set fileIds = new HashSet<>(); + + for (String tableName : tables) { + String selectSql = "SELECT DISTINCT file_id FROM " + tableName + " WHERE file_id IS NOT NULL"; + try (PreparedStatement select = connection.prepareStatement(selectSql); ResultSet resultSet = select.executeQuery()) { + while (resultSet.next()) { + String fileId = resultSet.getString("file_id"); + fileIds.add(fileId); + } + } + } + + return fileIds; + } + + + private Map buildFileIdToDossierTemplateIdMap(Connection connection, Set fileIds) throws SQLException { + + Map map = new HashMap<>(); + + if (!fileIds.isEmpty()) { + + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("SELECT f.id AS file_id, d.dossier_template_id "); + sqlBuilder.append("FROM file f "); + sqlBuilder.append("JOIN dossier d ON f.dossier_id = d.id "); + sqlBuilder.append("WHERE f.id IN ("); + + String placeholders = String.join(",", Collections.nCopies(fileIds.size(), "?")); + sqlBuilder.append(placeholders); + sqlBuilder.append(")"); + + String selectSql = sqlBuilder.toString(); + try (PreparedStatement select = connection.prepareStatement(selectSql)) { + int index = 1; + for (String fileId : fileIds) { + select.setString(index++, fileId); + } + try (ResultSet rs = select.executeQuery()) { + while (rs.next()) { + String fileId = rs.getString("file_id"); + String dossierTemplateId = rs.getString("dossier_template_id"); + map.put(fileId, dossierTemplateId); + } + } + } + } + + return map; + } + + + private Map buildReasonDossierTemplateIdToTechnicalNameMap(Connection connection) throws SQLException { + + Map map = new HashMap<>(); + + String selectSql = "SELECT reason, legal_basis_mapping_entity_dossier_template_id, technical_name FROM legal_basis_mapping_entity_legal_basis"; + try (PreparedStatement select = connection.prepareStatement(selectSql); ResultSet resultSet = select.executeQuery()) { + while (resultSet.next()) { + String reason = resultSet.getString("reason"); + String dossierTemplateId = resultSet.getString("legal_basis_mapping_entity_dossier_template_id"); + String technicalName = resultSet.getString("technical_name"); + + if (reason != null && dossierTemplateId != null && technicalName != null) { + String key = getKey(reason, dossierTemplateId); + map.put(key, technicalName); + } + } + } + + return map; + } + + + private int[] updateLegalBasisForTable(Connection connection, + String tableName, + HashFunction hashFunction, + Map fileIdToDossierTemplateIdMap, + Map reasonDossierTemplateIdToTechnicalNameMap) throws SQLException { + + PreparedStatement select = null; + PreparedStatement update = null; + ResultSet resultSet = null; + + int rowsUpdatedWithTechnicalName = 0; + int rowsHashed = 0; + + try { + String selectSql = "SELECT annotation_id, file_id, legal_basis FROM " + tableName + " WHERE legal_basis IS NOT NULL AND TRIM(legal_basis) <> '';"; + select = connection.prepareStatement(selectSql); + resultSet = select.executeQuery(); + + String updateSql = "UPDATE " + tableName + " SET legal_basis = ? WHERE annotation_id = ? AND file_id = ?"; + update = connection.prepareStatement(updateSql); + + while (resultSet.next()) { + String annotationId = resultSet.getString("annotation_id"); + String fileId = resultSet.getString("file_id"); + String originalValue = resultSet.getString("legal_basis"); + + if (originalValue == null) { + continue; + } + + String dossierTemplateId = fileIdToDossierTemplateIdMap.get(fileId); + + String newValue = null; + + if (dossierTemplateId != null) { + String key = getKey(originalValue, dossierTemplateId); + newValue = reasonDossierTemplateIdToTechnicalNameMap.get(key); + } + + if (newValue == null || newValue.trim().isEmpty()) { + newValue = hashFunction.hashString(originalValue, StandardCharsets.UTF_8).toString(); + rowsHashed++; + } else { + rowsUpdatedWithTechnicalName++; + } + + update.setString(1, newValue); + update.setString(2, annotationId); + update.setString(3, fileId); + update.executeUpdate(); + } + + } finally { + if (resultSet != null) { + resultSet.close(); + } + if (select != null) { + select.close(); + } + if (update != null) { + update.close(); + } + } + + return new int[]{rowsUpdatedWithTechnicalName, rowsHashed}; + } + + + private static String getKey(String originalValue, String dossierTemplateId) { + + return originalValue + "|" + dossierTemplateId; + } + + + @Override + public String getConfirmationMessage() { + + return "Technical name change applied to legalbasis fields in specified tables."; + } + + + @Override + public void setUp() {} + + + @Override + public void setFileOpener(ResourceAccessor resourceAccessor) {} + + + @Override + public ValidationErrors validate(Database database) { + + return new ValidationErrors(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V17MigrateImportedRedactionsFiles.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V17MigrateImportedRedactionsFiles.java index fa7f48efb..3f1c05c31 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V17MigrateImportedRedactionsFiles.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V17MigrateImportedRedactionsFiles.java @@ -48,7 +48,7 @@ public class V17MigrateImportedRedactionsFiles extends Migration { if (fileManagementStorageService.objectExists(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS)) { - com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions oldImportedRedactions = fileManagementStorageService.getImportedRedactions( + com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions oldImportedRedactions = fileManagementStorageService.getOldImportedRedactions( file.getDossierId(), file.getId()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V29TechnicalNameEntityLogMigration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V29TechnicalNameEntityLogMigration.java new file mode 100644 index 000000000..1c4343d9c --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V29TechnicalNameEntityLogMigration.java @@ -0,0 +1,151 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.migration.LegalBasisMappingService; +import com.iqser.red.service.persistence.management.v1.processor.migration.LegalBasisMappingService.TechnicalNameResult; +import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class V29TechnicalNameEntityLogMigration extends Migration { + + private final DossierRepository dossierRepository; + private final FileRepository fileRepository; + private final EntityLogMongoService entityLogMongoService; + private final LegalBasisMappingService legalBasisMappingService; + + private static final String NAME = "Migration for technical name update in entity logs"; + private static final long VERSION = 29; + + + public V29TechnicalNameEntityLogMigration(DossierRepository dossierRepository, + FileRepository fileRepository, + EntityLogMongoService entityLogMongoService, + LegalBasisMappingService legalBasisMappingService) { + + super(NAME, VERSION); + this.dossierRepository = dossierRepository; + this.fileRepository = fileRepository; + this.entityLogMongoService = entityLogMongoService; + this.legalBasisMappingService = legalBasisMappingService; + } + + + @Override + protected void migrate() { + + AtomicInteger totalEntriesProcessed = new AtomicInteger(0); + AtomicInteger totalValuesHashed = new AtomicInteger(0); + AtomicInteger totalValuesUpdatedNormally = new AtomicInteger(0); + + List dossiers = dossierRepository.findAll(); + + for (DossierEntity dossier : dossiers) { + String dossierId = dossier.getId(); + String dossierTemplateId = dossier.getDossierTemplateId(); + + + Map reasonToTechnicalNameMap = legalBasisMappingService.getReasonToTechnicalNameMap(dossierTemplateId); + + List files = fileRepository.findByDossierId(dossierId); + + for (FileEntity file : files) { + String fileId = file.getId(); + + Optional optionalEntityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId); + + if (optionalEntityLog.isPresent()) { + EntityLog entityLog = optionalEntityLog.get(); + + List legalBasisList = entityLog.getLegalBasis(); + if (legalBasisList == null) { + legalBasisList = new java.util.ArrayList<>(); + entityLog.setLegalBasis(legalBasisList); + } + + boolean updated = false; + + for (EntityLogEntry entry : entityLog.getEntityLogEntry()) { + totalEntriesProcessed.getAndIncrement(); + String currentLegalBasis = entry.getLegalBasis(); + + if (currentLegalBasis != null && !currentLegalBasis.isEmpty()) { + TechnicalNameResult result = legalBasisMappingService.processLegalBasisAndEntityLogLegalBasisList(currentLegalBasis, + reasonToTechnicalNameMap, + legalBasisList); + + if (result != null && !currentLegalBasis.equals(result.technicalName())) { + entry.setLegalBasis(result.technicalName()); + updated = true; + + if (result.hashed()) { + totalValuesHashed.getAndIncrement(); + } else { + totalValuesUpdatedNormally.getAndIncrement(); + } + } + } + + if (entry.getChanges() != null) { + for (Change change : entry.getChanges()) { + updated |= updatePropertyChanges(change.getPropertyChanges(), reasonToTechnicalNameMap, legalBasisList); + } + } + + if (entry.getManualChanges() != null) { + for (ManualChange manualChange : entry.getManualChanges()) { + updated |= updatePropertyChanges(manualChange.getPropertyChanges(), reasonToTechnicalNameMap, legalBasisList); + } + } + } + + if (updated) { + entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog); + log.info("Updated EntityLog for dossierId: {}, fileId: {}", dossierId, fileId); + } + } + } + } + + log.info("Migration completed."); + log.info("Total entries processed: {}", totalEntriesProcessed.get()); + log.info("Total values updated normally: {}", totalValuesUpdatedNormally.get()); + log.info("Total values hashed: {}", totalValuesHashed.get()); + } + + + private boolean updatePropertyChanges(Map propertyChanges, Map reasonToTechnicalNameMap, List legalBasisList) { + + boolean updated = false; + if (propertyChanges != null && propertyChanges.containsKey("legalBasis")) { + String legalBasisValue = propertyChanges.get("legalBasis"); + + String updatedLegalBasisValue = legalBasisMappingService.processTextAndEntityLogLegalBasisList(legalBasisValue, reasonToTechnicalNameMap, legalBasisList); + + if (!legalBasisValue.equals(updatedLegalBasisValue)) { + propertyChanges.put("legalBasis", updatedLegalBasisValue); + updated = true; + } + } + return updated; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V30TechnicalNameRuleFileMigration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V30TechnicalNameRuleFileMigration.java new file mode 100644 index 000000000..86405cbab --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V30TechnicalNameRuleFileMigration.java @@ -0,0 +1,118 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; +import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +@Slf4j +@Setter +@Service +public class V30TechnicalNameRuleFileMigration extends Migration { + + private final DossierTemplateRepository dossierTemplateRepository; + private final RulesPersistenceService rulesPersistenceService; + private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + + private static final String NAME = "Migration for replacing legal basis reasons with technical names in rule files"; + private static final long VERSION = 30; + + + public V30TechnicalNameRuleFileMigration(DossierTemplateRepository dossierTemplateRepository, + RulesPersistenceService rulesPersistenceService, + LegalBasisMappingPersistenceService legalBasisMappingPersistenceService) { + + super(NAME, VERSION); + this.dossierTemplateRepository = dossierTemplateRepository; + this.rulesPersistenceService = rulesPersistenceService; + this.legalBasisMappingPersistenceService = legalBasisMappingPersistenceService; + } + + + @Override + protected void migrate() { + + log.info("Migration: Updating rule files by replacing legal basis reasons with technical names"); + updateRuleFiles(); + } + + + private void updateRuleFiles() { + + List dossierTemplates = dossierTemplateRepository.findAll(); + String tenantId = TenantContext.getTenantId(); + + dossierTemplates.parallelStream() + .forEach(dossierTemplate -> { + TenantContext.setTenantId(tenantId); + String dossierTemplateId = dossierTemplate.getId(); + + List legalBasisMappings = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId); + + if (legalBasisMappings == null || legalBasisMappings.isEmpty()) { + log.warn("No legal basis mappings found for dossierTemplateId: {}", dossierTemplateId); + return; + } + + Map reasonToTechnicalNameMap = legalBasisMappings.stream() + .filter(lb -> StringUtils.isNotBlank(lb.getReason()) && StringUtils.isNotBlank(lb.getTechnicalName()) && !lb.getReason().equals(lb.getTechnicalName())) + .collect(Collectors.toMap(LegalBasisEntity::getReason, LegalBasisEntity::getTechnicalName, (existing, replacement) -> replacement)); + + if (reasonToTechnicalNameMap.isEmpty()) { + log.warn("No valid mappings to replace for dossierTemplateId: {}", dossierTemplateId); + return; + } + + Optional optionalRuleSet = rulesPersistenceService.getRules(dossierTemplateId, RuleFileType.ENTITY); + + if (optionalRuleSet.isPresent()) { + String originalRulesContent = optionalRuleSet.get().getValue(); + String updatedRulesContent = replaceReasonsWithTechnicalNames(originalRulesContent, reasonToTechnicalNameMap); + + if (!updatedRulesContent.equals(originalRulesContent)) { + rulesPersistenceService.setRules(updatedRulesContent, dossierTemplateId, RuleFileType.ENTITY); + log.info("Updated rule file for dossierTemplateId: {}", dossierTemplateId); + } else { + log.info("No replacements made for dossierTemplateId: {}", dossierTemplateId); + } + } else { + log.warn("No rule set found for dossierTemplateId: {}", dossierTemplateId); + } + }); + } + + + private String replaceReasonsWithTechnicalNames(String rulesContent, Map reasonToTechnicalNameMap) { + + String rulesContentResult = rulesContent; + for (Map.Entry entry : reasonToTechnicalNameMap.entrySet()) { + String reason = entry.getKey(); + String technicalName = entry.getValue(); + rulesContentResult = StringUtils.replace(rulesContentResult, quoteString(reason), quoteString(technicalName)); + } + return rulesContentResult; + } + + + private static String quoteString(String string) { + + return "\"" + string + "\""; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V31TechnicalNameImportedFilesMigration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V31TechnicalNameImportedFilesMigration.java new file mode 100644 index 000000000..b3e07c3f1 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V31TechnicalNameImportedFilesMigration.java @@ -0,0 +1,128 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.migration.LegalBasisMappingService; +import com.iqser.red.service.persistence.management.v1.processor.migration.LegalBasisMappingService.TechnicalNameResult; +import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class V31TechnicalNameImportedFilesMigration extends Migration { + + private static final String NAME = "Migration for replacing legal basis reasons with technical names in imported redactions and legal bases files"; + private static final long VERSION = 31; + + private final DossierRepository dossierRepository; + private final FileRepository fileRepository; + private final FileManagementStorageService fileManagementStorageService; + private final LegalBasisMappingService legalBasisMappingService; + + + public V31TechnicalNameImportedFilesMigration(DossierRepository dossierRepository, + FileRepository fileRepository, + FileManagementStorageService fileManagementStorageService, + LegalBasisMappingService legalBasisMappingService) { + + super(NAME, VERSION); + this.dossierRepository = dossierRepository; + this.fileRepository = fileRepository; + this.fileManagementStorageService = fileManagementStorageService; + this.legalBasisMappingService = legalBasisMappingService; + } + + + @Override + protected void migrate() { + + log.info("Migration: Updating imported redactions and legal bases files by replacing legal basis reasons with technical names"); + + List dossiers = dossierRepository.findAll(); + + for (DossierEntity dossier : dossiers) { + String dossierId = dossier.getId(); + String dossierTemplateId = dossier.getDossierTemplateId(); + + Map reasonToTechnicalNameMap = legalBasisMappingService.getReasonToTechnicalNameMap(dossierTemplateId); + + List files = fileRepository.findByDossierId(dossierId); + + for (FileEntity file : files) { + String fileId = file.getId(); + + if (fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_REDACTIONS)) { + ImportedRedactionsPerPage importedRedactions = fileManagementStorageService.getImportedRedactions(dossierId, fileId); + + boolean updated = false; + + for (ImportedRedaction redaction : importedRedactions.getImportedRedactions().values().stream().flatMap(Collection::stream).toList()) { + String currentLegalBasis = redaction.getLegalBasis(); + + if (currentLegalBasis != null && !currentLegalBasis.isEmpty()) { + TechnicalNameResult result = legalBasisMappingService.processLegalBasis(currentLegalBasis, reasonToTechnicalNameMap); + + if (result != null && !currentLegalBasis.equals(result.technicalName())) { + redaction.setLegalBasis(result.technicalName()); + updated = true; + } + } + + } + + if (updated) { + fileManagementStorageService.storeJSONObject(dossierId, fileId, FileType.IMPORTED_REDACTIONS, importedRedactions); + log.info("Updated imported redactions for dossierId: {}, fileId: {}", dossierId, fileId); + } + } + + if (fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES)) { + ImportedLegalBases importedLegalBases = fileManagementStorageService.getImportedLegalBases(dossierId, fileId); + + boolean updated = false; + + List legalBases = importedLegalBases.getImportedLegalBases(); + + for (ImportedLegalBasis importedLegalBasis : legalBases) { + String currentReason = importedLegalBasis.getReason(); + String currentTechnicalName = importedLegalBasis.getTechnicalName(); + + if (currentTechnicalName == null || currentTechnicalName.isEmpty()) { + if (currentReason != null && !currentReason.isEmpty()) { + TechnicalNameResult result = legalBasisMappingService.processLegalBasis(currentReason, reasonToTechnicalNameMap); + + if (result != null) { + importedLegalBasis.setTechnicalName(result.technicalName()); + updated = true; + } + } + } + } + + if (updated) { + fileManagementStorageService.storeJSONObject(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES, importedLegalBases); + log.info("Updated imported legal bases for dossierId: {}, fileId: {}", dossierId, fileId); + } + } + } + } + + log.info("Migration completed."); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ApprovalVerificationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ApprovalVerificationService.java index ac6c63d8c..d1d2265ec 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ApprovalVerificationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ApprovalVerificationService.java @@ -51,7 +51,7 @@ public class ApprovalVerificationService { addWarning(entry, WarningType.LEGAL_BASIS_MISSING, approveResponse); } else { var legalBasisEntity = legalBasisMappings.stream() - .filter(mapping -> mapping.getReason().equals(entry.getLegalBasis())) + .filter(mapping -> mapping.getTechnicalName().equals(entry.getLegalBasis())) .findFirst(); if (legalBasisEntity.isEmpty() || StringUtils.isEmpty(legalBasisEntity.get().getTechnicalName())) { addWarning(entry, WarningType.UNMAPPED_JUSTIFICATION, approveResponse); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java index 552e84aa5..824e4caca 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java @@ -239,10 +239,11 @@ public class DictionaryManagementService { @Transactional public void addEntries(String typeId, List entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType) { + addEntries(typeId, entries, removeCurrent, ignoreInvalidEntries, dictionaryEntryType, false); } - - + + @Transactional public void addEntries(String typeId, List entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType, boolean isImport) { @@ -293,7 +294,10 @@ public class DictionaryManagementService { // check for the existence of dossier type and create in case it does not exist if (isDossierTypeId(typeId)) { try { - dictionaryPersistenceService.getType(typeId); + TypeEntity type = dictionaryPersistenceService.getType(typeId, true); + if (type.isDeleted()) { + dictionaryPersistenceService.undeleteType(typeId); + } } catch (NotFoundException e) { // type not found check first dossier is matching the specified dossier template var dossierId = getDossierIdFromTypeId(typeId); @@ -397,9 +401,13 @@ public class DictionaryManagementService { dictionaryPersistenceService.deleteType(typeId); - entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY); - entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); - entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); + if (isDossierTypeId(typeId)) { + entryPersistenceService.hardDeleteAllEntriesForTypeId(typeId); + } else { + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY); + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); + } dictionaryPersistenceService.incrementVersion(typeId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java index d2ad83315..6739d394d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java @@ -268,7 +268,7 @@ public class DictionaryService { @PreAuthorize("hasAuthority('" + DELETE_DOSSIER_DICTIONARY_TYPE + "')") public void deleteDossierType(String type, String dossierTemplateId, String dossierId) { - accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId); + accessControlService.checkAccessPermissionsToDossier(dossierId); accessControlService.verifyUserIsMemberOrApprover(dossierId); deleteType(toTypeId(type, dossierTemplateId, dossierId)); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java index d7cd03ac8..9029f4ff7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java @@ -13,17 +13,17 @@ import java.util.List; import org.apache.commons.io.IOUtils; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; -import com.iqser.red.storage.commons.exception.StorageException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper; @@ -170,6 +170,7 @@ public class FileManagementStorageService { return entityLogMongoService.entityLogDocumentExists(dossierId, fileId); } + public boolean componentLogExists(String dossierId, String fileId) { return componentLogMongoService.componentLogDocumentExists(dossierId, fileId); @@ -189,12 +190,12 @@ public class FileManagementStorageService { } - public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { + public ImportedRedactionsPerPage getImportedRedactions(String dossierId, String fileId) { try { return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), - ImportedRedactions.class); + ImportedRedactionsPerPage.class); } catch (StorageObjectDoesNotExist e) { throw new NotFoundException("ImportedRedactions does not exist"); } catch (Exception e) { @@ -202,11 +203,41 @@ public class FileManagementStorageService { } } + + public ImportedLegalBases getImportedLegalBases(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES), + ImportedLegalBases.class); + } catch (StorageObjectDoesNotExist e) { + throw new NotFoundException("ImportedLegalBases does not exist"); + } catch (Exception e) { + throw new RuntimeException("Could not convert ImportedLegalBases", e); + } + } + + + public com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions getOldImportedRedactions(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), + com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions.class); + } catch (StorageObjectDoesNotExist e) { + throw new NotFoundException("ImportedRedactions does not exist"); + } catch (Exception e) { + throw new RuntimeException("Could not convert ImportedRedactions", e); + } + } + + public boolean objectExists(String dossierId, String fileId, String fileName, String fileExtension) { return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileName, fileExtension)); } + public boolean objectExists(String dossierId, String fileId, FileType origin) { return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, origin)); @@ -224,11 +255,13 @@ public class FileManagementStorageService { storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType)); } + public void deleteObject(String dossierId, String fileId, String fileName, String fileExtension) { storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileName, fileExtension)); } + public void deleteDocumentAndNerObjects(String dossierId, String fileId) { deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); @@ -249,6 +282,7 @@ public class FileManagementStorageService { } + public void deleteAllObjects(String dossierId, String fileId) { deleteObject(dossierId, fileId, FileType.VIEWER_DOCUMENT); @@ -270,6 +304,7 @@ public class FileManagementStorageService { entityLogMongoService.deleteEntityLog(dossierId, fileId); } + public void deleteComponentLog(String dossierId, String fileId) { componentLogMongoService.deleteComponentLog(dossierId, fileId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 41f51d8bb..b707ad2d4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -264,7 +264,7 @@ public class FileStatusService { var fileEntity = fileStatusPersistenceService.getStatus(fileId); if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN)) { - addToPreprocessingQueue(dossierId, fileId, fileEntity.getFilename()); + addToPreprocessingQueue(dossier.getDossierTemplateId(), dossierId, fileId, fileEntity.getFilename()); sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity); return; } @@ -487,9 +487,10 @@ public class FileStatusService { @SneakyThrows - public void addToPreprocessingQueue(String dossierId, String fileId, String filename) { + public void addToPreprocessingQueue(String dossierTemplateId, String dossierId, String fileId, String filename) { var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder() + .dossierTemplateId(dossierTemplateId) .highlightExtractionEnabled(currentApplicationTypeProvider.isRedactManager()) .dossierId(dossierId) .fileId(fileId) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index 47c6f2011..d66358877 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -10,6 +10,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Confl import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -109,7 +110,11 @@ public class ReanalysisService { private void analyseFiles(boolean repeatStructureAnalysis, List filesToReanalyse, boolean runOcr) { - filesToReanalyse.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, repeatStructureAnalysis, runOcr)); + filesToReanalyse.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), + file.getId(), + filesToReanalyse.size() == 1, + repeatStructureAnalysis, + runOcr)); } @@ -281,7 +286,11 @@ public class ReanalysisService { List rejectedFiles = filterInvalidFiles(files); - files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis(), reanalysisSettings.runOcr())); + files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), + file.getId(), + false, + reanalysisSettings.repeatStructureAnalysis(), + reanalysisSettings.runOcr())); return rejectedFiles; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java index e3952250c..987694a43 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java @@ -184,6 +184,13 @@ public class EntryPersistenceService { } + @Transactional + public void hardDeleteAllEntriesForTypeId(String typeId) { + + entryRepository.hardDeleteAllEntriesForTypeId(typeId); + } + + public void cloneEntries(String originalTypeId, String newTypeId) { entryRepository.cloneEntries(originalTypeId, newTypeId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java index 6ecb0ae37..00d5e5089 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java @@ -86,7 +86,7 @@ public interface TypeRepository extends JpaRepository { void softDeleteTypeById(@Param("typeId") String typeId); - @Modifying + @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("Update TypeEntity t set t.softDeletedTime = null where t.id = :typeId and t.softDeletedTime is not null") int unSoftDeleteTypeById(@Param("typeId") String typeId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java index d1892996f..69170a652 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java @@ -29,6 +29,11 @@ public interface EntryRepository extends EntryRepositoryCustom, JpaRepository updatedValueCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor annotationIdCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor fileIdCaptor = ArgumentCaptor.forClass(String.class); + + verify(mockUpdatePs, times(tables.length * 2)).setString(eq(1), updatedValueCaptor.capture()); + verify(mockUpdatePs, times(tables.length * 2)).setString(eq(2), annotationIdCaptor.capture()); + verify(mockUpdatePs, times(tables.length * 2)).setString(eq(3), fileIdCaptor.capture()); + + List updatedValues = updatedValueCaptor.getAllValues(); + List annotationIds = annotationIdCaptor.getAllValues(); + + HashFunction hashFunction = Hashing.murmur3_128(); + Map expectedUpdates = createExpectedUpdates(hashFunction); + + for (int i = 0; i < annotationIds.size(); i++) { + String annotationId = annotationIds.get(i); + String expectedValue = expectedUpdates.get(annotationId); + assertEquals(expectedValue, updatedValues.get(i), "Mismatch for annotation_id: " + annotationId); + } + } + + + @Test + public void testExecute_withException() { + + when(mockDatabase.getConnection()).thenThrow(new RuntimeException("Connection error")); + + CustomChangeException exception = assertThrows(CustomChangeException.class, () -> { + technicalNameChange.execute(mockDatabase); + }); + + assertTrue(exception.getMessage().contains("Error applying technical name change")); + assertEquals("Connection error", exception.getCause().getMessage()); + } + + + @Test + public void testExecute_noFileIds() throws Exception { + + try (PreparedStatement ps = mockSelectFileIdsPs) { + when(mockConnection.prepareStatement(startsWith("SELECT DISTINCT file_id FROM"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockSelectFileIdsRs); + when(mockSelectFileIdsRs.next()).thenReturn(false); + } + + technicalNameChange.execute(mockDatabase); + + verify(mockConnection, never()).prepareStatement(startsWith("SELECT f.id AS file_id, d.dossier_template_id")); + verify(mockConnection, never()).prepareStatement(startsWith("SELECT reason, legal_basis_mapping_entity_dossier_template_id, technical_name")); + + verify(mockConnection, never()).prepareStatement(startsWith("SELECT annotation_id, file_id, legal_basis FROM")); + verify(mockConnection, never()).prepareStatement(startsWith("UPDATE ")); + verify(mockUpdatePs, never()).executeUpdate(); + } + + + @Test + public void testExecute_noDossierMappings() throws Exception { + + String[] tables = {"manual_legal_basis_change", "manual_force_redaction", "manual_recategorization", "manual_redaction"}; + + try (PreparedStatement ps = mockSelectFileIdsPs) { + when(mockConnection.prepareStatement(startsWith("SELECT DISTINCT file_id FROM"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockSelectFileIdsRs); + when(mockSelectFileIdsRs.next()).thenReturn(true, false); + when(mockSelectFileIdsRs.getString("file_id")).thenReturn("file1"); + } + + try (PreparedStatement ps = mockDossierPs) { + when(mockConnection.prepareStatement(startsWith("SELECT f.id AS file_id, d.dossier_template_id"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockDossierRs); + when(mockDossierRs.next()).thenReturn(false); + } + + try (PreparedStatement ps = mockReasonPs) { + when(mockConnection.prepareStatement(startsWith("SELECT reason, legal_basis_mapping_entity_dossier_template_id, technical_name"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockReasonRs); + when(mockReasonRs.next()).thenReturn(false); + } + + for (String table : tables) { + try (PreparedStatement mockSelectUpdatePs = mock(PreparedStatement.class); ResultSet mockSelectUpdateRs = mock(ResultSet.class); PreparedStatement mockUpdate = mock( + PreparedStatement.class)) { + when(mockConnection.prepareStatement(startsWith("SELECT annotation_id, file_id, legal_basis FROM " + table))).thenReturn(mockSelectUpdatePs); + when(mockSelectUpdatePs.executeQuery()).thenReturn(mockSelectUpdateRs); + when(mockSelectUpdateRs.next()).thenReturn(true, false); + when(mockSelectUpdateRs.getString("annotation_id")).thenReturn("ann1"); + when(mockSelectUpdateRs.getString("file_id")).thenReturn("file1"); + when(mockSelectUpdateRs.getString("legal_basis")).thenReturn("reason1"); + + when(mockConnection.prepareStatement(startsWith("UPDATE " + table))).thenReturn(mockUpdate); + when(mockUpdate.executeUpdate()).thenReturn(1); + } + } + + technicalNameChange.execute(mockDatabase); + + verify(mockUpdatePs, never()).executeUpdate(); + } + + + @Test + public void testExecute_allTechnicalNamesPresent() throws Exception { + + String[] tables = {"manual_legal_basis_change", "manual_force_redaction", "manual_recategorization", "manual_redaction"}; + + try (PreparedStatement ps = mockSelectFileIdsPs) { + when(mockConnection.prepareStatement(startsWith("SELECT DISTINCT file_id FROM"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockSelectFileIdsRs); + when(mockSelectFileIdsRs.next()).thenReturn(true, false); + when(mockSelectFileIdsRs.getString("file_id")).thenReturn("file1"); + } + + try (PreparedStatement ps = mockDossierPs) { + when(mockConnection.prepareStatement(startsWith("SELECT f.id AS file_id, d.dossier_template_id"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockDossierRs); + when(mockDossierRs.next()).thenReturn(true, false); + when(mockDossierRs.getString("file_id")).thenReturn("file1"); + when(mockDossierRs.getString("dossier_template_id")).thenReturn("template1"); + } + + try (PreparedStatement ps = mockReasonPs) { + when(mockConnection.prepareStatement(startsWith("SELECT reason, legal_basis_mapping_entity_dossier_template_id, technical_name"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockReasonRs); + when(mockReasonRs.next()).thenReturn(true, false); + when(mockReasonRs.getString("reason")).thenReturn("reason1"); + when(mockReasonRs.getString("legal_basis_mapping_entity_dossier_template_id")).thenReturn("template1"); + when(mockReasonRs.getString("technical_name")).thenReturn("techName1"); + } + + for (String table : tables) { + try (PreparedStatement mockSelectUpdatePs = mock(PreparedStatement.class); ResultSet mockSelectUpdateRs = mock(ResultSet.class); PreparedStatement mockUpdate = mock( + PreparedStatement.class)) { + when(mockConnection.prepareStatement(startsWith("SELECT annotation_id, file_id, legal_basis FROM " + table))).thenReturn(mockSelectUpdatePs); + when(mockSelectUpdatePs.executeQuery()).thenReturn(mockSelectUpdateRs); + when(mockSelectUpdateRs.next()).thenReturn(true, false); + when(mockSelectUpdateRs.getString("annotation_id")).thenReturn("ann1"); + when(mockSelectUpdateRs.getString("file_id")).thenReturn("file1"); + when(mockSelectUpdateRs.getString("legal_basis")).thenReturn("reason1"); + + when(mockConnection.prepareStatement(startsWith("UPDATE " + table))).thenReturn(mockUpdate); + when(mockUpdate.executeUpdate()).thenReturn(1); + } + } + + technicalNameChange.execute(mockDatabase); + + verify(mockUpdatePs, never()).executeUpdate(); + } + + + @Test + public void testExecute_sqlExceptionDuringUpdate() throws Exception { + + try (PreparedStatement ps = mockSelectFileIdsPs) { + when(mockConnection.prepareStatement(startsWith("SELECT DISTINCT file_id FROM"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockSelectFileIdsRs); + when(mockSelectFileIdsRs.next()).thenReturn(true, false); + when(mockSelectFileIdsRs.getString("file_id")).thenReturn("file1"); + } + + try (PreparedStatement ps = mockDossierPs) { + when(mockConnection.prepareStatement(startsWith("SELECT f.id AS file_id, d.dossier_template_id"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockDossierRs); + when(mockDossierRs.next()).thenReturn(true, false); + when(mockDossierRs.getString("file_id")).thenReturn("file1"); + when(mockDossierRs.getString("dossier_template_id")).thenReturn("template1"); + } + + try (PreparedStatement ps = mockReasonPs) { + when(mockConnection.prepareStatement(startsWith("SELECT reason, legal_basis_mapping_entity_dossier_template_id, technical_name"))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(mockReasonRs); + when(mockReasonRs.next()).thenReturn(true, false); + when(mockReasonRs.getString("reason")).thenReturn("reason1"); + when(mockReasonRs.getString("legal_basis_mapping_entity_dossier_template_id")).thenReturn("template1"); + when(mockReasonRs.getString("technical_name")).thenReturn("techName1"); + } + + try (PreparedStatement mockSelectUpdatePs = mock(PreparedStatement.class); ResultSet mockSelectUpdateRs = mock(ResultSet.class)) { + when(mockConnection.prepareStatement(startsWith("SELECT annotation_id, file_id, legal_basis FROM manual_legal_basis_change"))).thenReturn(mockSelectUpdatePs); + when(mockSelectUpdatePs.executeQuery()).thenReturn(mockSelectUpdateRs); + when(mockSelectUpdateRs.next()).thenReturn(true, false); + when(mockSelectUpdateRs.getString("annotation_id")).thenReturn("ann1"); + when(mockSelectUpdateRs.getString("file_id")).thenReturn("file1"); + when(mockSelectUpdateRs.getString("legal_basis")).thenReturn("reason1"); + } + + try (PreparedStatement mockUpdate = mock(PreparedStatement.class)) { + when(mockConnection.prepareStatement(startsWith("UPDATE manual_legal_basis_change"))).thenReturn(mockUpdate); + when(mockUpdate.executeUpdate()).thenThrow(new SQLException("Update failed")); + } + + CustomChangeException exception = assertThrows(CustomChangeException.class, () -> { + technicalNameChange.execute(mockDatabase); + }); + + assertTrue(exception.getMessage().contains("Error applying technical name change")); + assertInstanceOf(SQLException.class, exception.getCause()); + assertEquals("Update failed", exception.getCause().getMessage()); + } + + + private void setupTableMocks(PreparedStatement ps, + ResultSet rs, + String tableName, + String annotationId1, + String annotationId2, + String fileId1, + String fileId2, + String legalBasis1, + String legalBasis2) throws SQLException { + + when(mockConnection.prepareStatement(startsWith("SELECT annotation_id, file_id, legal_basis FROM " + tableName))).thenReturn(ps); + when(ps.executeQuery()).thenReturn(rs); + when(rs.next()).thenReturn(true, true, false); + when(rs.getString("annotation_id")).thenReturn(annotationId1, annotationId2); + when(rs.getString("file_id")).thenReturn(fileId1, fileId2); + when(rs.getString("legal_basis")).thenReturn(legalBasis1, legalBasis2); + } + + + private Map createExpectedUpdates(HashFunction hashFunction) { + + Map expectedUpdates = new HashMap<>(); + expectedUpdates.put("ann1", "techName1"); + expectedUpdates.put("ann2", hashFunction.hashString("reason2", StandardCharsets.UTF_8).toString()); + expectedUpdates.put("ann3", hashFunction.hashString("reason3", StandardCharsets.UTF_8).toString()); + expectedUpdates.put("ann4", hashFunction.hashString("reason4", StandardCharsets.UTF_8).toString()); + expectedUpdates.put("ann5", hashFunction.hashString("reason5", StandardCharsets.UTF_8).toString()); + expectedUpdates.put("ann6", hashFunction.hashString("reason6", StandardCharsets.UTF_8).toString()); + expectedUpdates.put("ann7", hashFunction.hashString("reason7", StandardCharsets.UTF_8).toString()); + expectedUpdates.put("ann8", hashFunction.hashString("reason8", StandardCharsets.UTF_8).toString()); + return expectedUpdates; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameEntityLogMigrationTest.java b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameEntityLogMigrationTest.java new file mode 100644 index 000000000..cd011a484 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameEntityLogMigrationTest.java @@ -0,0 +1,636 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.V29TechnicalNameEntityLogMigration; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; + +public class TechnicalNameEntityLogMigrationTest { + + @Mock + private DossierRepository dossierRepository; + + @Mock + private FileRepository fileRepository; + + @Mock + private EntityLogMongoService entityLogMongoService; + + @Mock + private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + + @Mock + private MigrationPersistenceService migrationPersistenceService; + + private V29TechnicalNameEntityLogMigration migration; + + private final HashFunction hashFunction = Hashing.murmur3_128(); + + + @BeforeEach + public void setup() { + + MockitoAnnotations.openMocks(this); + + when(migrationPersistenceService.getLatestProcessedVersion()).thenReturn(0L); + when(migrationPersistenceService.isProcessed(anyLong(), anyLong())).thenReturn(false); + + LegalBasisMappingService legalBasisMappingService = new LegalBasisMappingService(legalBasisMappingPersistenceService); + + migration = new V29TechnicalNameEntityLogMigration(dossierRepository, fileRepository, entityLogMongoService, legalBasisMappingService); + + migration.setMigrationPersistenceService(migrationPersistenceService); + } + + + @Test + public void testMigrationUpdatesMappedLegalBasisTechnicalNames() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_TechName").reason("LB1_Reason").build(); + LegalBasisEntity lb2 = LegalBasisEntity.builder().technicalName("LB2_TechName").reason("LB2_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Arrays.asList(lb1, lb2)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + + EntityLogEntry entry1 = EntityLogEntry.builder().legalBasis("LB1_Reason").build(); + EntityLogEntry entry2 = EntityLogEntry.builder().legalBasis("LB2_Reason").build(); + EntityLog entityLog = createEntityLog(Arrays.asList(entry1, entry2)); + + entityLog.setLegalBasis(List.of(EntityLogLegalBasis.builder().technicalName("LB1_TechName").reason("LB1_Reason").build(), + EntityLogLegalBasis.builder().technicalName("LB2_TechName").reason("LB2_Reason").build())); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + assertAll(() -> assertEquals("LB1_TechName", entry1.getLegalBasis()), () -> assertEquals("LB2_TechName", entry2.getLegalBasis())); + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + } + + + @Test + public void testMigrationHashesUnmappedLegalBasis() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.emptyList()); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + + EntityLogEntry entry1 = EntityLogEntry.builder().legalBasis("Unmapped_LB1").build(); + EntityLogEntry entry2 = EntityLogEntry.builder().legalBasis("Unmapped_LB2").build(); + EntityLog entityLog = createEntityLog(Arrays.asList(entry1, entry2)); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + String hashedLB1 = hashFunction.hashString("Unmapped_LB1", StandardCharsets.UTF_8).toString(); + String hashedLB2 = hashFunction.hashString("Unmapped_LB2", StandardCharsets.UTF_8).toString(); + assertAll(() -> assertEquals(hashedLB1, entry1.getLegalBasis()), () -> assertEquals(hashedLB2, entry2.getLegalBasis())); + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + } + + + @Test + public void testMigrationSkipsEntriesWithoutLegalBasis() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.emptyList()); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + + EntityLogEntry entry1 = EntityLogEntry.builder().legalBasis(null).build(); + EntityLogEntry entry2 = EntityLogEntry.builder().legalBasis("").build(); + EntityLog entityLog = createEntityLog(Arrays.asList(entry1, entry2)); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + assertAll(() -> assertNull(entry1.getLegalBasis()), () -> assertEquals("", entry2.getLegalBasis())); + verify(entityLogMongoService, never()).saveEntityLog(anyString(), anyString(), any(EntityLog.class)); + } + + + @Test + public void testMigrationSkipsFilesWithoutEntityLog() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.empty()); + + migration.run(); + + verify(entityLogMongoService, never()).saveEntityLog(anyString(), anyString(), any(EntityLog.class)); + } + + + @Test + public void testMigrationWithNoDossiers() { + + when(dossierRepository.findAll()).thenReturn(Collections.emptyList()); + + migration.run(); + + verify(fileRepository, never()).findByDossierId(anyString()); + verify(entityLogMongoService, never()).findEntityLogByDossierIdAndFileId(anyString(), anyString()); + } + + + @Test + public void testMigrationWithMixedEntries() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + + LegalBasisEntity mappedLB = LegalBasisEntity.builder().technicalName("Mapped_TechName").reason("Mapped_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(List.of(mappedLB)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + + EntityLogEntry mappedEntry = EntityLogEntry.builder().legalBasis("Mapped_Reason").build(); + EntityLogEntry unmappedEntry = EntityLogEntry.builder().legalBasis("Unmapped_Reason").build(); + EntityLogEntry nullEntry = EntityLogEntry.builder().legalBasis(null).build(); + EntityLog entityLog = createEntityLog(Arrays.asList(mappedEntry, unmappedEntry, nullEntry)); + + entityLog.setLegalBasis(new ArrayList<>()); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + String hashedUnmapped = hashFunction.hashString("Unmapped_Reason", StandardCharsets.UTF_8).toString(); + + assertAll(() -> assertEquals("Mapped_TechName", mappedEntry.getLegalBasis()), + () -> assertEquals(hashedUnmapped, unmappedEntry.getLegalBasis()), + () -> assertNull(nullEntry.getLegalBasis())); + + assertEquals(2, entityLog.getLegalBasis().size()); + assertTrue(entityLog.getLegalBasis() + .stream() + .anyMatch(lb -> lb.getTechnicalName().equals("Mapped_TechName") && lb.getReason().equals("Mapped_Reason"))); + assertTrue(entityLog.getLegalBasis() + .stream() + .anyMatch(lb -> lb.getTechnicalName().equals(hashedUnmapped) && lb.getReason().equals("Unmapped_Reason"))); + + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + } + + + @Test + public void testMigrationDoesNotUpdateWhenNoChanges() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.singletonList(LegalBasisEntity.builder() + .technicalName("LB1_Tech") + .reason("LB1_Reason") + .build())); + + FileEntity file = createFile("file1", "dossier1"); + + EntityLogEntry entry = EntityLogEntry.builder().legalBasis("LB1_Tech").build(); + EntityLog entityLog = createEntityLog(List.of(entry)); + + entityLog.setLegalBasis(new ArrayList<>(List.of(EntityLogLegalBasis.builder().technicalName("LB1_Tech").reason("LB1_Reason").build()))); + + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + verify(entityLogMongoService, never()).saveEntityLog(anyString(), anyString(), any(EntityLog.class)); + } + + + @Test + public void testMigrationProcessesMultipleDossiers() { + + DossierEntity dossier1 = createDossier("dossier1", "template1"); + DossierEntity dossier2 = createDossier("dossier2", "template2"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier1, dossier2)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_Tech").reason("LB1_Reason").build(); + LegalBasisEntity lb2 = LegalBasisEntity.builder().technicalName("LB2_Tech").reason("LB2_Reason").build(); + LegalBasisEntity lb3 = LegalBasisEntity.builder().technicalName("LB3_Tech").reason("LB3_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(List.of(lb1, lb2)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template2")).thenReturn(List.of(lb3)); + + FileEntity file1 = createFile("file1", "dossier1"); + FileEntity file2 = createFile("file2", "dossier2"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file1)); + when(fileRepository.findByDossierId("dossier2")).thenReturn(List.of(file2)); + + EntityLogEntry entry1 = EntityLogEntry.builder().legalBasis("LB1_Reason").build(); + EntityLog entityLog1 = createEntityLog(List.of(entry1)); + entityLog1.setLegalBasis(List.of(EntityLogLegalBasis.builder().technicalName("LB1_Tech").reason("LB1_Reason").build(), + EntityLogLegalBasis.builder().technicalName("LB2_Tech").reason("LB2_Reason").build())); + + EntityLogEntry entry2 = EntityLogEntry.builder().legalBasis("LB3_Reason").build(); + EntityLog entityLog2 = createEntityLog(List.of(entry2)); + entityLog2.setLegalBasis(List.of(EntityLogLegalBasis.builder().technicalName("LB3_Tech").reason("LB3_Reason").build())); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog1)); + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier2", "file2")).thenReturn(Optional.of(entityLog2)); + + migration.run(); + + assertAll(() -> assertEquals("LB1_Tech", entry1.getLegalBasis()), () -> assertEquals("LB3_Tech", entry2.getLegalBasis())); + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog1); + verify(entityLogMongoService).saveEntityLog("dossier2", "file2", entityLog2); + } + + + @Test + public void testMigrationProcessesMultipleFilesPerDossier() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_Tech").reason("LB1_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(List.of(lb1)); + + FileEntity file1 = createFile("file1", "dossier1"); + FileEntity file2 = createFile("file2", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file1, file2)); + + EntityLogEntry entry1 = EntityLogEntry.builder().legalBasis("LB1_Reason").build(); + EntityLog entityLog1 = createEntityLog(List.of(entry1)); + entityLog1.setLegalBasis(List.of(EntityLogLegalBasis.builder().technicalName("LB1_Tech").reason("LB1_Reason").build())); + + EntityLogEntry entry2 = EntityLogEntry.builder().legalBasis("Unmapped_LB").build(); + EntityLog entityLog2 = createEntityLog(List.of(entry2)); + entityLog2.setLegalBasis(new ArrayList<>()); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog1)); + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file2")).thenReturn(Optional.of(entityLog2)); + + migration.run(); + + String expectedHash = hashFunction.hashString("Unmapped_LB", StandardCharsets.UTF_8).toString(); + + assertAll(() -> assertEquals("LB1_Tech", entry1.getLegalBasis()), () -> assertEquals(expectedHash, entry2.getLegalBasis())); + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog1); + verify(entityLogMongoService).saveEntityLog("dossier1", "file2", entityLog2); + } + + + @Test + public void testMigrationDoesNotAddDuplicateLegalBasis() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.emptyList()); + + FileEntity file = createFile("file1", "dossier1"); + + String hashedLB = hashFunction.hashString("Unmapped_LB", StandardCharsets.UTF_8).toString(); + EntityLogLegalBasis existingLB = createEntityLogLegalBasisForUnmappedLegalBasis(hashedLB); + + EntityLogEntry entry = EntityLogEntry.builder().legalBasis("Unmapped_LB").build(); + EntityLog entityLog = createEntityLog(List.of(entry)); + entityLog.setLegalBasis(new ArrayList<>(List.of(existingLB))); + + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + assertEquals(hashedLB, entry.getLegalBasis()); + assertEquals(1, entityLog.getLegalBasis().size()); + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + } + + + @Test + public void testMigrationInitializesLegalBasisListWhenNull() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.emptyList()); + + FileEntity file = createFile("file1", "dossier1"); + + EntityLogEntry entry = EntityLogEntry.builder().legalBasis("Unmapped_LB").build(); + EntityLog entityLog = createEntityLog(List.of(entry)); + entityLog.setLegalBasis(null); + + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + assertNotNull(entityLog.getLegalBasis()); + assertEquals(1, entityLog.getLegalBasis().size()); + + String expectedHash = hashFunction.hashString("Unmapped_LB", StandardCharsets.UTF_8).toString(); + assertEquals(expectedHash, entry.getLegalBasis()); + + EntityLogLegalBasis newLB = entityLog.getLegalBasis() + .get(0); + assertEquals(expectedHash, newLB.getTechnicalName()); + assertEquals("Unmapped_LB", newLB.getReason()); + assertEquals("Generated Legal Basis", newLB.getName()); + assertEquals("", newLB.getDescription()); + + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + } + + + @Test + public void testMigrationUpdates() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_Tech").reason("LB1_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(List.of(lb1)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + + EntityLogEntry entry1 = EntityLogEntry.builder().legalBasis("LB1_Reason").build(); + EntityLogEntry entry2 = EntityLogEntry.builder().legalBasis("Unmapped_LB").build(); + EntityLogEntry entry3 = EntityLogEntry.builder().legalBasis("Another_Unmapped_LB").build(); + EntityLog entityLog = createEntityLog(Arrays.asList(entry1, entry2, entry3)); + + entityLog.setLegalBasis(new ArrayList<>()); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + String hashedUnmappedLB = hashFunction.hashString("Unmapped_LB", StandardCharsets.UTF_8).toString(); + String hashedAnotherUnmappedLB = hashFunction.hashString("Another_Unmapped_LB", StandardCharsets.UTF_8).toString(); + + assertAll(() -> assertEquals("LB1_Tech", entry1.getLegalBasis()), + () -> assertEquals(hashedUnmappedLB, entry2.getLegalBasis()), + () -> assertEquals(hashedAnotherUnmappedLB, entry3.getLegalBasis())); + + assertEquals(3, entityLog.getLegalBasis().size()); + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + } + + + @Test + public void testMigrationHandlesDuplicateEntityLogEntries() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_Tech").reason("LB1_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(List.of(lb1)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + + EntityLogEntry entry1 = EntityLogEntry.builder().legalBasis("LB1_Reason").build(); + EntityLogEntry entry2 = EntityLogEntry.builder().legalBasis("LB1_Reason").build(); + EntityLog entityLog = createEntityLog(List.of(entry1, entry2)); + + entityLog.setLegalBasis(new ArrayList<>()); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + assertAll(() -> assertEquals("LB1_Tech", entry1.getLegalBasis()), + () -> assertEquals("LB1_Tech", entry2.getLegalBasis()), + () -> assertEquals(1, entityLog.getLegalBasis().size())); + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + } + + + @Test + public void testMigrationHandlesExistingHashedTechnicalName() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.emptyList()); + + FileEntity file = createFile("file1", "dossier1"); + + String hashedLB = hashFunction.hashString("Unmapped_LB", StandardCharsets.UTF_8).toString(); + EntityLogLegalBasis existingLB = createEntityLogLegalBasisForUnmappedLegalBasis(hashedLB); + + EntityLogEntry entry = EntityLogEntry.builder().legalBasis("Unmapped_LB").build(); + EntityLog entityLog = createEntityLog(List.of(entry)); + entityLog.setLegalBasis(new ArrayList<>(List.of(existingLB))); + + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + assertEquals(hashedLB, entry.getLegalBasis()); + assertEquals(1, entityLog.getLegalBasis().size()); + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + } + + + @Test + public void testMigrationSkipsSavingWhenNoEntriesUpdated() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_Tech").reason("LB1_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(List.of(lb1)); + + FileEntity file = createFile("file1", "dossier1"); + + EntityLogEntry entry = EntityLogEntry.builder().legalBasis("LB1_Tech").build(); + EntityLogLegalBasis lb = EntityLogLegalBasis.builder() + .technicalName("LB1_Tech") + .reason("LB1_Reason") + .name("Generated Legal Basis") + .description("") + .build(); + EntityLog entityLog = createEntityLog(List.of(entry)); + entityLog.setLegalBasis(new ArrayList<>(List.of(lb))); + + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + verify(entityLogMongoService, never()).saveEntityLog(anyString(), anyString(), any(EntityLog.class)); + } + + + @Test + public void testMigrationUpdatesPropertyChangesLegalBasis() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(List.of(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("personal_data_geolocation_article_39e3").reason("Article 39(e)(3) of Regulation (EC) No 178/2002").build(); + + LegalBasisEntity lb2 = LegalBasisEntity.builder() + .technicalName("article_63_2_a_regulation_1107_2009") + .reason("Article 63(2)(a) of Regulation (EC) No 1107/2009​‌‌​ (making reference to Article 39 of Regulation EC No 178/2002)") + .build(); + + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(List.of(lb1, lb2)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(List.of(file)); + + Change change1 = new Change(); + change1.setAnalysisNumber(18); + change1.setType(ChangeType.CHANGED); + change1.setDateTime(OffsetDateTime.now()); + + Map propertyChanges1 = new HashMap<>(); + propertyChanges1.put("legalBasis", "personal_data_geolocation_article_39e3 -> Article 39(e)(3) of Regulation (EC) No 178/2002"); + change1.setPropertyChanges(propertyChanges1); + + Change change2 = new Change(); + change2.setAnalysisNumber(19); + change2.setType(ChangeType.CHANGED); + change2.setDateTime(OffsetDateTime.now()); + + Map propertyChanges2 = new HashMap<>(); + propertyChanges2.put("legalBasis", "Article 39(e)(3) of Regulation (EC) No 178/2002 -> personal_data_geolocation_article_39e3"); + change2.setPropertyChanges(propertyChanges2); + + ManualChange manualChange = new ManualChange(); + manualChange.setManualRedactionType(ManualRedactionType.RECATEGORIZE); + manualChange.setProcessedDate(OffsetDateTime.now()); + manualChange.setRequestedDate(OffsetDateTime.now()); + manualChange.setUserId("user123"); + + Map manualPropertyChanges = new HashMap<>(); + manualPropertyChanges.put("legalBasis", "Article 63(2)(a) of Regulation (EC) No 1107/2009​‌‌​ (making reference to Article 39 of Regulation EC No 178/2002)"); + manualChange.setPropertyChanges(manualPropertyChanges); + + EntityLogEntry entry = EntityLogEntry.builder() + .legalBasis("personal_data_geolocation_article_39e3") + .changes(List.of(change1, change2)) + .manualChanges(List.of(manualChange)) + .build(); + + EntityLog entityLog = createEntityLog(List.of(entry)); + entityLog.setLegalBasis(new ArrayList<>()); + + when(entityLogMongoService.findEntityLogByDossierIdAndFileId("dossier1", "file1")).thenReturn(Optional.of(entityLog)); + + migration.run(); + + String expectedLegalBasis1 = "personal_data_geolocation_article_39e3 -> personal_data_geolocation_article_39e3"; + String expectedLegalBasis2 = "personal_data_geolocation_article_39e3 -> personal_data_geolocation_article_39e3"; + String expectedManualLegalBasis = "article_63_2_a_regulation_1107_2009"; + + assertEquals(expectedLegalBasis1, + change1.getPropertyChanges() + .get("legalBasis")); + assertEquals(expectedLegalBasis2, + change2.getPropertyChanges() + .get("legalBasis")); + assertEquals(expectedManualLegalBasis, + manualChange.getPropertyChanges() + .get("legalBasis")); + + verify(entityLogMongoService).saveEntityLog("dossier1", "file1", entityLog); + + assertEquals(2, entityLog.getLegalBasis().size()); + assertTrue(entityLog.getLegalBasis() + .stream() + .anyMatch(lb -> lb.getTechnicalName().equals("personal_data_geolocation_article_39e3"))); + assertTrue(entityLog.getLegalBasis() + .stream() + .anyMatch(lb -> lb.getTechnicalName().equals("article_63_2_a_regulation_1107_2009"))); + } + + + private DossierEntity createDossier(String id, String templateId) { + + DossierEntity dossier = new DossierEntity(); + dossier.setId(id); + dossier.setDossierTemplateId(templateId); + return dossier; + } + + + private FileEntity createFile(String id, String dossierId) { + + FileEntity file = new FileEntity(); + file.setId(id); + file.setDossierId(dossierId); + return file; + } + + + private EntityLog createEntityLog(List entries) { + + EntityLog entityLog = new EntityLog(); + entityLog.setEntityLogEntry(entries); + return entityLog; + } + + + private EntityLogLegalBasis createEntityLogLegalBasisForUnmappedLegalBasis(String technicalName) { + + return EntityLogLegalBasis.builder() + .technicalName(technicalName) + .reason("Unmapped_LB") + .name("Generated Legal Basis") + .description("") + .build(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameImportedFilesMigrationTest.java b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameImportedFilesMigrationTest.java new file mode 100644 index 000000000..13bbb0aaa --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameImportedFilesMigrationTest.java @@ -0,0 +1,366 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.V31TechnicalNameImportedFilesMigration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; + +public class TechnicalNameImportedFilesMigrationTest { + + @Mock + private DossierRepository dossierRepository; + + @Mock + private FileRepository fileRepository; + + @Mock + private FileManagementStorageService fileManagementStorageService; + + @Mock + private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + + @Mock + private MigrationPersistenceService migrationPersistenceService; + + private LegalBasisMappingService legalBasisMappingService; + + private V31TechnicalNameImportedFilesMigration migration; + + + @BeforeEach + public void setup() { + + MockitoAnnotations.openMocks(this); + + when(migrationPersistenceService.getLatestProcessedVersion()).thenReturn(0L); + when(migrationPersistenceService.isProcessed(anyLong(), anyLong())).thenReturn(false); + + legalBasisMappingService = new LegalBasisMappingService(legalBasisMappingPersistenceService); + + migration = new V31TechnicalNameImportedFilesMigration(dossierRepository, fileRepository, fileManagementStorageService, legalBasisMappingService); + + migration.setMigrationPersistenceService(migrationPersistenceService); + } + + + @Test + public void testMigrationUpdatesMappedLegalBasisTechnicalNamesInImportedRedactions() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(Collections.singletonList(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_TechName").reason("LB1_Reason").build(); + LegalBasisEntity lb2 = LegalBasisEntity.builder().technicalName("LB2_TechName").reason("LB2_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Arrays.asList(lb1, lb2)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.singletonList(file)); + + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_REDACTIONS)).thenReturn(true); + + ImportedRedaction redaction1 = new ImportedRedaction(); + redaction1.setLegalBasis("LB1_Reason"); + + ImportedRedaction redaction2 = new ImportedRedaction(); + redaction2.setLegalBasis("LB2_Reason"); + + ImportedRedactionsPerPage importedRedactions = new ImportedRedactionsPerPage(); + importedRedactions.setImportedRedactions(Map.of(1, Arrays.asList(redaction1, redaction2))); + + when(fileManagementStorageService.getImportedRedactions("dossier1", "file1")).thenReturn(importedRedactions); + + migration.run(); + + assertEquals("LB1_TechName", redaction1.getLegalBasis()); + assertEquals("LB2_TechName", redaction2.getLegalBasis()); + + verify(fileManagementStorageService).storeJSONObject("dossier1", "file1", FileType.IMPORTED_REDACTIONS, importedRedactions); + } + + + @Test + public void testMigrationHashesUnmappedLegalBasisInImportedRedactions() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(Collections.singletonList(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.emptyList()); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.singletonList(file)); + + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_REDACTIONS)).thenReturn(true); + + ImportedRedaction redaction1 = new ImportedRedaction(); + redaction1.setLegalBasis("Unmapped_LB1"); + + ImportedRedaction redaction2 = new ImportedRedaction(); + redaction2.setLegalBasis("Unmapped_LB2"); + + ImportedRedactionsPerPage importedRedactions = new ImportedRedactionsPerPage(); + importedRedactions.setImportedRedactions(Map.of(1, Arrays.asList(redaction1, redaction2))); + + when(fileManagementStorageService.getImportedRedactions("dossier1", "file1")).thenReturn(importedRedactions); + + migration.run(); + + String hashedLB1 = legalBasisMappingService.processLegalBasis("Unmapped_LB1", new HashMap<>()).technicalName(); + String hashedLB2 = legalBasisMappingService.processLegalBasis("Unmapped_LB2", new HashMap<>()).technicalName(); + + assertEquals(hashedLB1, redaction1.getLegalBasis()); + assertEquals(hashedLB2, redaction2.getLegalBasis()); + + verify(fileManagementStorageService).storeJSONObject("dossier1", "file1", FileType.IMPORTED_REDACTIONS, importedRedactions); + } + + + @Test + public void testMigrationUpdatesTechnicalNamesInImportedLegalBases() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(Collections.singletonList(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_TechName").reason("LB1_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.singletonList(lb1)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.singletonList(file)); + + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_LEGAL_BASES)).thenReturn(true); + + ImportedLegalBasis importedLegalBasis = new ImportedLegalBasis(); + importedLegalBasis.setReason("LB1_Reason"); + importedLegalBasis.setTechnicalName(null); + + ImportedLegalBases importedLegalBases = new ImportedLegalBases(); + importedLegalBases.setImportedLegalBases(Collections.singletonList(importedLegalBasis)); + + when(fileManagementStorageService.getImportedLegalBases("dossier1", "file1")).thenReturn(importedLegalBases); + + migration.run(); + + assertEquals("LB1_TechName", importedLegalBasis.getTechnicalName()); + + verify(fileManagementStorageService).storeJSONObject("dossier1", "file1", FileType.IMPORTED_LEGAL_BASES, importedLegalBases); + } + + + @Test + public void testMigrationHandlesFilesWithoutImportedRedactionsOrLegalBases() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(Collections.singletonList(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.emptyList()); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.singletonList(file)); + + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_REDACTIONS)).thenReturn(false); + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_LEGAL_BASES)).thenReturn(false); + + migration.run(); + + verify(fileManagementStorageService, never()).getImportedRedactions(anyString(), anyString()); + verify(fileManagementStorageService, never()).getImportedLegalBases(anyString(), anyString()); + + verify(fileManagementStorageService, never()).storeJSONObject(anyString(), anyString(), any(FileType.class), any()); + } + + + @Test + public void testMigrationSkipsDossiersWithNoFiles() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(Collections.singletonList(dossier)); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.emptyList()); + + migration.run(); + + verify(fileManagementStorageService, never()).objectExists(anyString(), anyString(), any(FileType.class)); + } + + + @Test + public void testMigrationProcessesMultipleDossiersAndFiles() { + + DossierEntity dossier1 = createDossier("dossier1", "template1"); + DossierEntity dossier2 = createDossier("dossier2", "template2"); + when(dossierRepository.findAll()).thenReturn(Arrays.asList(dossier1, dossier2)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_TechName").reason("LB1_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.singletonList(lb1)); + + LegalBasisEntity lb2 = LegalBasisEntity.builder().technicalName("LB2_TechName").reason("LB2_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template2")).thenReturn(Collections.singletonList(lb2)); + + FileEntity file1 = createFile("file1", "dossier1"); + FileEntity file2 = createFile("file2", "dossier2"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.singletonList(file1)); + when(fileRepository.findByDossierId("dossier2")).thenReturn(Collections.singletonList(file2)); + + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_REDACTIONS)).thenReturn(true); + when(fileManagementStorageService.objectExists("dossier2", "file2", FileType.IMPORTED_REDACTIONS)).thenReturn(true); + + ImportedRedaction redaction1 = new ImportedRedaction(); + redaction1.setLegalBasis("LB1_Reason"); + + ImportedRedactionsPerPage importedRedactions1 = new ImportedRedactionsPerPage(); + importedRedactions1.setImportedRedactions(Map.of(1, Collections.singletonList(redaction1))); + + when(fileManagementStorageService.getImportedRedactions("dossier1", "file1")).thenReturn(importedRedactions1); + + ImportedRedaction redaction2 = new ImportedRedaction(); + redaction2.setLegalBasis("LB2_Reason"); + + ImportedRedactionsPerPage importedRedactions2 = new ImportedRedactionsPerPage(); + importedRedactions2.setImportedRedactions(Map.of(1, Collections.singletonList(redaction2))); + + when(fileManagementStorageService.getImportedRedactions("dossier2", "file2")).thenReturn(importedRedactions2); + + migration.run(); + + assertEquals("LB1_TechName", redaction1.getLegalBasis()); + assertEquals("LB2_TechName", redaction2.getLegalBasis()); + + verify(fileManagementStorageService).storeJSONObject("dossier1", "file1", FileType.IMPORTED_REDACTIONS, importedRedactions1); + verify(fileManagementStorageService).storeJSONObject("dossier2", "file2", FileType.IMPORTED_REDACTIONS, importedRedactions2); + } + + + @Test + public void testMigrationDoesNotUpdateWhenNoChangesNeeded() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(Collections.singletonList(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_TechName").reason("LB1_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.singletonList(lb1)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.singletonList(file)); + + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_REDACTIONS)).thenReturn(true); + + ImportedRedaction redaction = new ImportedRedaction(); + redaction.setLegalBasis("LB1_TechName"); + + ImportedRedactionsPerPage importedRedactions = new ImportedRedactionsPerPage(); + importedRedactions.setImportedRedactions(Map.of(1,Collections.singletonList(redaction))); + + when(fileManagementStorageService.getImportedRedactions("dossier1", "file1")).thenReturn(importedRedactions); + + migration.run(); + + assertEquals("LB1_TechName", redaction.getLegalBasis()); + + verify(fileManagementStorageService, never()).storeJSONObject(anyString(), anyString(), any(FileType.class), any()); + } + + + @Test + public void testMigrationHandlesImportedLegalBasesWithExistingTechnicalNames() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(Collections.singletonList(dossier)); + + LegalBasisEntity lb1 = LegalBasisEntity.builder().technicalName("LB1_TechName").reason("LB1_Reason").build(); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.singletonList(lb1)); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.singletonList(file)); + + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_LEGAL_BASES)).thenReturn(true); + + ImportedLegalBasis importedLegalBasis = new ImportedLegalBasis(); + importedLegalBasis.setReason("LB1_Reason"); + importedLegalBasis.setTechnicalName("LB1_TechName"); + + ImportedLegalBases importedLegalBases = new ImportedLegalBases(); + importedLegalBases.setImportedLegalBases(Collections.singletonList(importedLegalBasis)); + + when(fileManagementStorageService.getImportedLegalBases("dossier1", "file1")).thenReturn(importedLegalBases); + + migration.run(); + + assertEquals("LB1_TechName", importedLegalBasis.getTechnicalName()); + + verify(fileManagementStorageService, never()).storeJSONObject(anyString(), anyString(), any(FileType.class), any()); + } + + + @Test + public void testMigrationHashesUnmappedReasonsInImportedLegalBases() { + + DossierEntity dossier = createDossier("dossier1", "template1"); + when(dossierRepository.findAll()).thenReturn(Collections.singletonList(dossier)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping("template1")).thenReturn(Collections.emptyList()); + + FileEntity file = createFile("file1", "dossier1"); + when(fileRepository.findByDossierId("dossier1")).thenReturn(Collections.singletonList(file)); + + when(fileManagementStorageService.objectExists("dossier1", "file1", FileType.IMPORTED_LEGAL_BASES)).thenReturn(true); + + ImportedLegalBasis importedLegalBasis = new ImportedLegalBasis(); + importedLegalBasis.setReason("Unmapped_Reason"); + importedLegalBasis.setTechnicalName(null); + + ImportedLegalBases importedLegalBases = new ImportedLegalBases(); + importedLegalBases.setImportedLegalBases(Collections.singletonList(importedLegalBasis)); + + when(fileManagementStorageService.getImportedLegalBases("dossier1", "file1")).thenReturn(importedLegalBases); + + migration.run(); + + String hashedTechnicalName = legalBasisMappingService.processLegalBasis("Unmapped_Reason", new HashMap<>()).technicalName(); + + assertEquals(hashedTechnicalName, importedLegalBasis.getTechnicalName()); + + verify(fileManagementStorageService).storeJSONObject("dossier1", "file1", FileType.IMPORTED_LEGAL_BASES, importedLegalBases); + } + + + private DossierEntity createDossier(String id, String templateId) { + + DossierEntity dossier = new DossierEntity(); + dossier.setId(id); + dossier.setDossierTemplateId(templateId); + return dossier; + } + + + private FileEntity createFile(String id, String dossierId) { + + FileEntity file = new FileEntity(); + file.setId(id); + file.setDossierId(dossierId); + return file; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameRuleFileMigrationTest.java b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameRuleFileMigrationTest.java new file mode 100644 index 000000000..44d781e50 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/migration/TechnicalNameRuleFileMigrationTest.java @@ -0,0 +1,672 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.*; +import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; +import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.V30TechnicalNameRuleFileMigration; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; + +public class TechnicalNameRuleFileMigrationTest { + + private static final String TEMPLATE_ID_1 = "template1"; + private static final String TEMPLATE_ID_2 = "template2"; + + private static final String RULES_WITH_REASONS = "rule \"CBI.0.4: Redact CBI Authors (vertebrate Study)\"\n" + + " when\n" + + " FileAttribute(label == \"Vertebrate Study\", value soundslike \"Yes\" || value.toLowerCase() == \"y\")\n" + + " $entity: TextEntity(type() == \"CBI_author\", dictionaryEntry)\n" + + " then\n" + + " $entity.redact(\"CBI.0.4\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n" + + " end\n" + + "\n" + + "rule \"CBI.0.3: Redact CBI Authors (non vertebrate Study)\"\n" + + " when\n" + + " not FileAttribute(label == \"Vertebrate Study\", value soundslike \"Yes\" || value.toLowerCase() == \"y\")\n" + + " $entity: TextEntity(type() == \"CBI_author\", dictionaryEntry)\n" + + " then\n" + + " $entity.redact(\"CBI.0.3\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n" + + " end"; + + private static final String RULES_WITH_SPECIAL_CHARACTERS = "rule \"Special Characters Rule\"\n" + + " when\n" + + " // conditions\n" + + " then\n" + + " // actions\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)\");\n" + + " end"; + + private static final String RULES_WITH_NO_REASONS = "rule \"Some Rule\"\n" + " when\n" + " // conditions\n" + " then\n" + " // actions\n" + " end"; + + private static final String RULES_WITH_PARTIAL_MATCH = "rule \"Partial Match Rule\"\n" + + " when\n" + + " // conditions\n" + + " then\n" + + " // actions\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Some Article 39(e)(2) of Regulation (EC) No 178/2002 Additional Text\");\n" + + " end"; + + private static final String RULES_WITH_IDENTICAL_REASON_AND_TECHNICAL_NAME = "rule \"Identical Reason and Technical Name Rule\"\n" + + " when\n" + + " // conditions\n" + + " then\n" + + " // actions\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"same_name\");\n" + + " end"; + + private static final String RULES_WITH_MULTIPLE_TEMPLATES = "rule \"Composition Rule\"\n" + + " when\n" + + " // conditions\n" + + " then\n" + + " // actions\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Article 63(2)(d) of Regulation (EC) No 1107/2009\");\n" + + " end"; + + private static final String RULES_WITH_NULL_VALUES = "rule \"Null Values Rule\"\n" + + " when\n" + + " // conditions\n" + + " then\n" + + " // actions\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n" + + " end"; + + private static final String RULES_WITH_PARTIAL_VALID_MAPPINGS = "rule \"Partial Valid Mappings Rule\"\n" + + " when\n" + + " // conditions\n" + + " then\n" + + " // actions\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Some Other Reason\");\n" + + " end"; + + @Mock + private DossierTemplateRepository dossierTemplateRepository; + + @Mock + private RulesPersistenceService rulesPersistenceService; + + @Mock + private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + + @Mock + private MigrationPersistenceService migrationPersistenceService; + + @InjectMocks + private V30TechnicalNameRuleFileMigration migration; + + + @BeforeEach + public void setup() { + + MockitoAnnotations.openMocks(this); + migration.setMigrationPersistenceService(migrationPersistenceService); + when(migrationPersistenceService.getLatestProcessedVersion()).thenReturn(0L); + when(migrationPersistenceService.isProcessed(anyLong(), anyLong())).thenReturn(false); + } + + + private DossierTemplateEntity createDossierTemplate(String id) { + + DossierTemplateEntity dossierTemplate = new DossierTemplateEntity(); + dossierTemplate.setId(id); + return dossierTemplate; + } + + + private LegalBasisEntity createLegalBasis(String reason, String technicalName) { + + return LegalBasisEntity.builder().reason(reason).technicalName(technicalName).build(); + } + + + private RuleSetEntity createRuleSet(String content) { + + RuleSetEntity ruleSet = new RuleSetEntity(); + ruleSet.setValue(content); + return ruleSet; + } + + + @Test + public void testMigrationReplacesLegalBasisReasonsWithTechnicalNames() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + List mappings = Arrays.asList(createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", + "vertebrate_study_personal_data_geolocation_article_39e2"), + createLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002", "personal_data_geolocation_article_39e3")); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(mappings); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_REASONS); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedRules.contains("vertebrate_study_personal_data_geolocation_article_39e2")), + () -> assertTrue(updatedRules.contains("personal_data_geolocation_article_39e3")), + () -> assertFalse(updatedRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002")), + () -> assertFalse(updatedRules.contains("Article 39(e)(3) of Regulation (EC) No 178/2002"))); + } + + + @Test + public void testMigrationWithNoLegalBasisMappings() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.emptyList()); + + migration.run(); + + verifyNoInteractions(rulesPersistenceService); + } + + + @Test + public void testMigrationWithNoRuleSet() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "vertebrate_study_personal_data_geolocation_article_39e2"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.empty()); + + migration.run(); + + verify(rulesPersistenceService, never()).setRules(anyString(), anyString(), any(RuleFileType.class)); + } + + + @Test + public void testMigrationWithNoReplacementsNeeded() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Nonexistent Reason", "nonexistent_technical_name"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_NO_REASONS); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + verify(rulesPersistenceService, never()).setRules(anyString(), anyString(), any(RuleFileType.class)); + } + + + @Test + public void testMigrationHandlesSpecialCharactersInReasons() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)", + "manufacturing_production_process"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_SPECIAL_CHARACTERS); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedRules.contains("manufacturing_production_process")), + () -> assertFalse(updatedRules.contains("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"))); + } + + + @Test + public void testMigrationDoesNotReplacePartialMatches() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "vertebrate_study_personal_data_geolocation_article_39e2"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_PARTIAL_MATCH); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + verify(rulesPersistenceService, never()).setRules(anyString(), anyString(), any(RuleFileType.class)); + } + + + @Test + public void testMigrationSkipsIdenticalReasonAndTechnicalName() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("same_name", "same_name"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_IDENTICAL_REASON_AND_TECHNICAL_NAME); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + verify(rulesPersistenceService, never()).setRules(anyString(), anyString(), any(RuleFileType.class)); + } + + + @Test + public void testMigrationProcessesMultipleDossierTemplates() { + + DossierTemplateEntity dossierTemplate1 = createDossierTemplate(TEMPLATE_ID_1); + DossierTemplateEntity dossierTemplate2 = createDossierTemplate(TEMPLATE_ID_2); + when(dossierTemplateRepository.findAll()).thenReturn(Arrays.asList(dossierTemplate1, dossierTemplate2)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "vertebrate_study_personal_data_geolocation_article_39e2"); + LegalBasisEntity lb2 = createLegalBasis("Article 63(2)(d) of Regulation (EC) No 1107/2009", "composition_plant_protection_product"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_2)).thenReturn(Collections.singletonList(lb2)); + + RuleSetEntity ruleSet1 = createRuleSet(RULES_WITH_REASONS); + RuleSetEntity ruleSet2 = createRuleSet(RULES_WITH_MULTIPLE_TEMPLATES); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet1)); + when(rulesPersistenceService.getRules(TEMPLATE_ID_2, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet2)); + + migration.run(); + + ArgumentCaptor rulesCaptor1 = ArgumentCaptor.forClass(String.class); + ArgumentCaptor rulesCaptor2 = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService, times(2)).setRules(any(), any(), eq(RuleFileType.ENTITY)); + verify(rulesPersistenceService).setRules(rulesCaptor1.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + verify(rulesPersistenceService).setRules(rulesCaptor2.capture(), eq(TEMPLATE_ID_2), eq(RuleFileType.ENTITY)); + + String updatedRules1 = rulesCaptor1.getValue(); + String updatedRules2 = rulesCaptor2.getValue(); + + assertAll(() -> { + assertTrue(updatedRules1.contains("vertebrate_study_personal_data_geolocation_article_39e2")); + assertFalse(updatedRules1.contains("Article 39(e)(2) of Regulation (EC) No 178/2002")); + }, () -> { + assertTrue(updatedRules2.contains("composition_plant_protection_product")); + assertFalse(updatedRules2.contains("Article 63(2)(d) of Regulation (EC) No 1107/2009")); + }); + } + + + @Test + public void testMigrationHandlesNullValuesGracefully() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis(null, "personal_data_geolocation_article_39e3"); + LegalBasisEntity lb2 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", null); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Arrays.asList(lb1, lb2)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_NULL_VALUES); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + verify(rulesPersistenceService, never()).setRules(anyString(), anyString(), any(RuleFileType.class)); + } + + + @Test + public void testMigrationUpdatesRuleSetWhenOnlySomeMappingsAreValid() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "vertebrate_study_personal_data_geolocation_article_39e2"); + LegalBasisEntity lb2 = createLegalBasis(null, "some_technical_name"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Arrays.asList(lb1, lb2)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_PARTIAL_VALID_MAPPINGS); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedRules.contains("vertebrate_study_personal_data_geolocation_article_39e2")), + () -> assertFalse(updatedRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002")), + () -> assertTrue(updatedRules.contains("Some Other Reason"))); + } + + + @Test + public void testMigrationWithEmptyRulesContent() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "vertebrate_study_personal_data_geolocation_article_39e2"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + RuleSetEntity ruleSet = createRuleSet(""); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + verify(rulesPersistenceService, never()).setRules(anyString(), anyString(), any(RuleFileType.class)); + } + + + @Test + public void testMigrationReplacesMultipleOccurrencesOfSameReason() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "tech_name_39e2"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + String duplicatedRules = RULES_WITH_REASONS + "\n" + RULES_WITH_REASONS; + RuleSetEntity ruleSet = createRuleSet(duplicatedRules); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertEquals(2, StringUtils.countMatches(updatedRules, "tech_name_39e2")), + () -> assertFalse(updatedRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002"))); + } + + + @Test + public void testMigrationDoesNotReplaceOverlappingReasonStrings() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "tech_name_39e2"); + LegalBasisEntity lb2 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002 Extended", "tech_name_extended"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Arrays.asList(lb1, lb2)); + + String overlappingRules = "rule \"Test Rule\"\n" + + " when\n" + + " $entity: TextEntity(type() == \"CBI_author\", dictionaryEntry)\n" + + " then\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Article 39(e)(2) of Regulation (EC) No 178/2002 Extended\");\n" + + " end"; + + RuleSetEntity ruleSet = createRuleSet(overlappingRules); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedRules.contains("tech_name_39e2")), + () -> assertTrue(updatedRules.contains("tech_name_extended")), + () -> assertFalse(updatedRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002")), + () -> assertFalse(updatedRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002 Extended"))); + } + + + @Test + public void testMigrationWithDuplicateLegalBasisMappings() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "tech_name_39e2"); + LegalBasisEntity lb2 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "tech_name_39e2_duplicate"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Arrays.asList(lb1, lb2)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_REASONS); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedRules.contains("tech_name_39e2_duplicate")), + () -> assertFalse(updatedRules.contains("\"tech_name_39e2\"")), + () -> assertFalse(updatedRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002"))); + } + + + @Test + public void testMigrationWithTechnicalNamesContainingQuotes() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "tech_name_with_\"quotes\""); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_REASONS); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedRules.contains("tech_name_with_\"quotes\"")), + () -> assertFalse(updatedRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002"))); + } + + + @Test + public void testMigrationDoesNotAlterOtherRuleFileTypes() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "tech_name_39e2"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + RuleSetEntity entityRuleSet = createRuleSet(RULES_WITH_REASONS); + RuleSetEntity componentRuleSet = createRuleSet("component rules content"); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(entityRuleSet)); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.COMPONENT)).thenReturn(Optional.of(componentRuleSet)); + + migration.run(); + + ArgumentCaptor entityRulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(entityRulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedEntityRules = entityRulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedEntityRules.contains("tech_name_39e2")), + () -> assertFalse(updatedEntityRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002"))); + + verify(rulesPersistenceService, never()).setRules(eq("component rules content"), eq(TEMPLATE_ID_1), eq(RuleFileType.COMPONENT)); + } + + + @Test + public void testMigrationDoesNotReprocessAlreadyMigratedRuleSets() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + when(migrationPersistenceService.getLatestProcessedVersion()).thenReturn(28L); + + migration.run(); + + verifyNoInteractions(rulesPersistenceService); + } + + + @Test + public void testMigrationWithLargeRuleSetsAndMappings() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + List largeMappings = IntStream.range(0, 1000).mapToObj(i -> createLegalBasis("Article " + i + "(e)(2) of Regulation (EC) No 178/2002", "tech_name_" + i)) + .collect(Collectors.toList()); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(largeMappings); + + String largeRules = IntStream.range(0, 1000) + .mapToObj(i -> "rule \"Rule " + + i + + "\"\n" + + " when\n" + + " $entity: TextEntity(type() == \"CBI_author\", dictionaryEntry)\n" + + " then\n" + + " $entity.redact(\"RuleCode\", \"Description\", \"Article " + + i + + "(e)(2) of Regulation (EC) No 178/2002\");\n" + + " end\n\n") + .collect(Collectors.joining()); + + RuleSetEntity ruleSet = createRuleSet(largeRules); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedRules.contains("tech_name_0")), + () -> assertTrue(updatedRules.contains("tech_name_500")), + () -> assertTrue(updatedRules.contains("tech_name_999")), + () -> { + assertFalse(updatedRules.contains("Article 0(e)(2) of Regulation (EC) No 178/2002")); + assertFalse(updatedRules.contains("Article 500(e)(2) of Regulation (EC) No 178/2002")); + assertFalse(updatedRules.contains("Article 999(e)(2) of Regulation (EC) No 178/2002")); + }); + } + + + @Test + public void testMigrationSkipsMappingsWithMissingTechnicalNames() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", null); + LegalBasisEntity lb2 = createLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002", "tech_name_39e3"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Arrays.asList(lb1, lb2)); + + RuleSetEntity ruleSet = createRuleSet(RULES_WITH_REASONS); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + ArgumentCaptor rulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(rulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedRules = rulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedRules.contains("tech_name_39e3")), + () -> assertFalse(updatedRules.contains("Article 39(e)(3) of Regulation (EC) No 178/2002")), + () -> assertTrue(updatedRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002"))); + } + + + @Test + public void testMigrationWithEmptyLegalBasisMappingsList() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.emptyList()); + + migration.run(); + + verifyNoInteractions(rulesPersistenceService); + } + + + @Test + public void testMigrationProcessesOnlySpecifiedRuleFileType() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "tech_name_39e2"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + RuleSetEntity entityRuleSet = createRuleSet(RULES_WITH_REASONS); + RuleSetEntity componentRuleSet = createRuleSet("component rules content"); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(entityRuleSet)); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.COMPONENT)).thenReturn(Optional.of(componentRuleSet)); + + migration.run(); + + ArgumentCaptor entityRulesCaptor = ArgumentCaptor.forClass(String.class); + verify(rulesPersistenceService).setRules(entityRulesCaptor.capture(), eq(TEMPLATE_ID_1), eq(RuleFileType.ENTITY)); + + String updatedEntityRules = entityRulesCaptor.getValue(); + assertAll(() -> assertTrue(updatedEntityRules.contains("tech_name_39e2")), + () -> assertFalse(updatedEntityRules.contains("Article 39(e)(2) of Regulation (EC) No 178/2002"))); + + verify(rulesPersistenceService, never()).setRules(eq("component rules content"), eq(TEMPLATE_ID_1), eq(RuleFileType.COMPONENT)); + } + + + @Test + public void testMigrationWithInvalidRuleFileContent() { + + DossierTemplateEntity dossierTemplate = createDossierTemplate(TEMPLATE_ID_1); + when(dossierTemplateRepository.findAll()).thenReturn(Collections.singletonList(dossierTemplate)); + + LegalBasisEntity lb1 = createLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002", "tech_name_39e2"); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(TEMPLATE_ID_1)).thenReturn(Collections.singletonList(lb1)); + + String malformedRules = "rule \"Malformed Rule\"\n" + + " when\n" + + " $entity: TextEntity(type() == \"CBI_author\", dictionaryEntry)\n" + + " then\n" + + " $entity.redact(RuleCode, Description, Article 39(e)(2) of Regulation (EC) No 178/2002);\n" + + " end"; + + RuleSetEntity ruleSet = createRuleSet(malformedRules); + when(rulesPersistenceService.getRules(TEMPLATE_ID_1, RuleFileType.ENTITY)).thenReturn(Optional.of(ruleSet)); + + migration.run(); + + verify(rulesPersistenceService, never()).setRules(anyString(), anyString(), any(RuleFileType.class)); + } + + + @Test + public void testMigrationWithNoDossierTemplates() { + + when(dossierTemplateRepository.findAll()).thenReturn(Collections.emptyList()); + + migration.run(); + + verifyNoInteractions(rulesPersistenceService, legalBasisMappingPersistenceService); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java index be58780ec..657069ade 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java @@ -14,6 +14,7 @@ import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryC import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; import com.iqser.red.service.peristence.v1.server.integration.client.LegalBasisClient; import com.iqser.red.service.peristence.v1.server.integration.client.RulesClient; +import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType; @@ -36,6 +37,9 @@ public class DossierTemplateTesterAndProvider { @Autowired private DictionaryClient dictionaryClient; + @Autowired + private CurrentApplicationTypeProvider currentApplicationTypeProvider; + public Colors provideDefaultColors(String dossierTemplateId) { @@ -82,6 +86,9 @@ public class DossierTemplateTesterAndProvider { rulesClient.upload(new RulesUploadRequestModel("ABCD", loadedTemplate.getDossierTemplateId(), RuleFileType.ENTITY)); legalBasisClient.setLegalBasisMapping(List.of(new LegalBasis("name", "description", "reason","technicalName")), loadedTemplate.getDossierTemplateId()); + if (currentApplicationTypeProvider.isDocuMine()) { + rulesClient.upload(new RulesUploadRequestModel("ABCD", loadedTemplate.getDossierTemplateId(), RuleFileType.COMPONENT)); + } loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getDossierTemplateId()); return loadedTemplate; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ApprovalTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ApprovalTest.java index 89a0e472a..114b55c5e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ApprovalTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ApprovalTest.java @@ -70,7 +70,7 @@ public class ApprovalTest extends AbstractPersistenceServerServiceTest { .id("id1") .positions(List.of(new Position(1, 1, 1, 1, 1))) .state(EntryState.APPLIED) - .legalBasis("legalBasis") + .legalBasis("legal_basis") .entryType(EntryType.ENTITY) .value("value") .build())); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java index ef7542f32..c48c66905 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java @@ -64,8 +64,6 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest @Test - @Disabled - // todo: fix me public void testOverrides() throws IOException { var dossier = dossierTesterAndProvider.provideTestDossier(); @@ -187,8 +185,6 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest @Test @SneakyThrows - @Disabled - //todo: fix me public void testDeletedFileOverrides() throws IOException { var dossier = dossierTesterAndProvider.provideTestDossier(); @@ -257,7 +253,7 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest assertTrue(overrides.getComponentOverrides().isEmpty()); // case 2: re-upload file that was deleted before overrides should also not be returned anymore - fileTesterAndProvider.testAndProvideFile(dossier, "filename3"); + fileTesterAndProvider.testAndProvideFile(dossier, "filename2"); var e = assertThrows(FeignException.class, () -> componentClient.getOverrides(dossierTemplate.getId(), dossier.getId(), file.getId())); assertEquals(e.status(), 404); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java index 5aa8377c3..09d9d5698 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java @@ -20,13 +20,18 @@ import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; +import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository; import com.iqser.red.service.persistence.service.v1.api.external.model.UpdateEntries; import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue; @@ -60,6 +65,15 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { @Autowired private DossierTemplateClient dossierTemplateClient; + @Autowired + private DossierClient dossierClient; + + @Autowired + private TypeRepository typeRepository; + + @Autowired + private EntryRepository entryRepository; + private final List testList1 = List.of("William c. Spare", "Charalampos", "Carina Wilson", "PATRICIA A. SHEEHY", "William C. Spare", "carlsen", "Patricia A. Sheehy"); private final List testList2 = List.of("William C. Spare", "Charalampos", "Carina Wilson", "Patricia A. Sheehy", "William c. Spare", "carlsen", "PATRICIA A. SHEEHY"); @@ -409,19 +423,19 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { .build()); assertThat(returnedtype1.isDossierDictionaryOnly()).isTrue(); - assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), null, false) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(1); - assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier.getId(), false) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(2); + assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), null, false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(1); + assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier.getId(), false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(2); var dictionary = dictionaryClient.getDictionaryForType(returnedtype1.getType(), dossier.getDossierTemplateId(), dossier2.getId()); - assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier2.getId(), false) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(2); + assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier2.getId(), false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(2); } @@ -461,9 +475,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { assertThat(actualEntries).usingComparator(new ListContentWithoutOrderAndWithoutDuplicatesComparator<>()).isEqualTo(entries); dictionaryClient.deleteType(createdType.getType(), createdType.getDossierTemplateId()); - assertThat(dictionaryClient.getAllTypes(createdType.getDossierTemplateId(), null, false) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()).size()).isEqualTo(0); + assertThat(dictionaryClient.getAllTypes(createdType.getDossierTemplateId(), null, false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(0); dictionaryClient.addType(type); @@ -1019,20 +1034,20 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { .build()); assertThat(dtType.getRank()).isEqualTo(100); assertThat(dtType.isDossierDictionaryOnly()).isFalse(); - assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(1); + assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(1); dictionaryClient.deleteType(dtType.getType(), dtType.getDossierTemplateId()); - assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, false) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(0); - assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(1); + assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(0); + assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(1); var dtType2 = dictionaryClient.addType(CreateTypeValue.builder() .type("test2") @@ -1046,24 +1061,24 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { .dossierTemplateId(dossierTemplate.getId()) .dossierDictionaryOnly(false) .build()); - assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, false) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(1); - assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(2); + assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(1); + assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(2); assertThat(dtType2.getRank()).isEqualTo(dtType.getRank()); - assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(2); - assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), true) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(3); + assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(2); + assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), true).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(3); } @@ -1096,7 +1111,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { .dossierDictionaryOnly(false) .build()); - List types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes().stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()); + List types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()); assertThat(types.size()).isEqualTo(1); String dictionaryEntry = "entry1"; @@ -1129,10 +1147,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { .build()); assertThat(dtType.getRank()).isEqualTo(100); assertThat(dtType.isDossierDictionaryOnly()).isFalse(); - assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true) - .getTypes() - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(1); + assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .collect(Collectors.toList()).size()).isEqualTo(1); assertThatThrownBy(() -> dictionaryManagementService.addEntries(dtType.getTypeId(), List.of("entry1", "entry2"), @@ -1143,6 +1161,137 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { } + @Test + void testDeleteDossierThenDeleteType() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate, "TestDossier"); + + CreateTypeValue createTypeValue = CreateTypeValue.builder() + .type("test_type") + .label("Test Type") + .hexColor("#fcba03") + .rank(100) + .hint(false) + .caseInsensitive(false) + .recommendation(false) + .addToDictionaryAction(true) + .dossierTemplateId(dossierTemplate.getId()) + .dossierDictionaryOnly(false) + .build(); + + var type = dictionaryClient.addType(createTypeValue); + + List customTypes = dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .toList(); + + assertEquals(2, customTypes.size()); + + dossierClient.deleteDossier(dossier.getId()); + + dictionaryClient.deleteType(type.getType(), type.getDossierTemplateId()); + + var types = dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .toList(); + + assertThat(types).isEmpty(); + + dictionaryClient.addType(createTypeValue); + + customTypes = dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes() + .stream() + .filter(t -> !t.isSystemManaged()) + .toList(); + + assertEquals(2, customTypes.size()); + } + + + @Test + void testRecreateTypeAndCheckMergedDictionary() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate, "TestDossier"); + + var type = typeProvider.testAndProvideType(dossierTemplate, null, "type1", false); + + List templateEntries = List.of("aaa", "bbb"); + dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), templateEntries, false, null, DictionaryEntryType.ENTRY); + var templateDictionary = dictionaryClient.getDictionaryForType(type.getType(), dossierTemplate.getId(), null); + assertThat(templateDictionary.getEntries()).containsExactlyInAnyOrderElementsOf(templateEntries); + + List dossierEntries = List.of("ccc"); + dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), dossierEntries, false, dossier.getId(), DictionaryEntryType.ENTRY); + var dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), dossierTemplate.getId(), dossier.getId()); + assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrderElementsOf(dossierEntries); + + Dictionary mergedDictionary = dictionaryClient.getMergedDictionaries(type.getType(), type.getDossierTemplateId(), dossier.getId()); + assertThat(mergedDictionary).isNotNull(); + List allEntries = new ArrayList<>(templateEntries); + allEntries.addAll(dossierEntries); + assertThat(mergedDictionary.getEntries()).containsExactlyInAnyOrderElementsOf(allEntries); + + List newDossierEntries = List.of("entry d"); + UpdateEntries updateEntries = new UpdateEntries(newDossierEntries, allEntries); + dictionaryClient.updateEntries(type.getType(), type.getDossierTemplateId(), updateEntries, dossier.getId(), DictionaryEntryType.ENTRY); + + List deleted = entryRepository.findAll().stream().filter(DictionaryEntryEntity::isDeleted).toList(); + assertEquals(3, deleted.size()); + + var updatedDossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), dossierTemplate.getId(), dossier.getId()); + assertThat(updatedDossierDictionary.getEntries()).containsExactlyInAnyOrderElementsOf(newDossierEntries); + + // deletion + dictionaryClient.deleteType(type.getType(), type.getDossierTemplateId()); + Assertions.assertThrows(FeignException.NotFound.class, () -> dictionaryClient.getDictionaryForType(type.getType(), dossierTemplate.getId(), null)); + + List deletedType1Entities = typeRepository.findAllTypesByDossierTemplateIdOrDossierId(dossierTemplate.getId(), dossier.getId()) + .stream() + .filter(typeEntity -> typeEntity.getType().equals("type1") && typeEntity.isDeleted()) + .toList(); + + assertEquals(2, deletedType1Entities.size()); + + deleted = entryRepository.findAll().stream().filter(DictionaryEntryEntity::isDeleted).toList(); + assertEquals(2, deleted.size()); + + // recreation + var type2 = typeProvider.testAndProvideType(dossierTemplate, null, "type1", false); + + dictionaryClient.addEntry(type2.getType(), type2.getDossierTemplateId(), templateEntries, false, null, DictionaryEntryType.ENTRY); + + deleted = entryRepository.findAll().stream().filter(DictionaryEntryEntity::isDeleted).toList(); + assertEquals(0, deleted.size()); + + deletedType1Entities = typeRepository.findAllTypesByDossierTemplateIdOrDossierId(dossierTemplate.getId(), dossier.getId()) + .stream() + .filter(typeEntity -> typeEntity.getType().equals("type1") && typeEntity.isDeleted()) + .toList(); + + assertEquals(1, deletedType1Entities.size()); + + // problematic call because it creates/update entities on a get + dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false); + + deletedType1Entities = typeRepository.findAllTypesByDossierTemplateIdOrDossierId(dossierTemplate.getId(), dossier.getId()) + .stream() + .filter(typeEntity -> typeEntity.getType().equals("type1") && typeEntity.isDeleted()) + .toList(); + + assertEquals(0, deletedType1Entities.size()); + + Dictionary mergedDictionary2 = dictionaryClient.getMergedDictionaries(type2.getType(), type2.getDossierTemplateId(), dossier.getId()); + assertThat(mergedDictionary2).isNotNull(); + assertThat(mergedDictionary2.getEntries()).containsExactlyInAnyOrderElementsOf(templateEntries); + assertThat(mergedDictionary2.getEntries()).doesNotContain("entry d"); + } + + private static final class ListContentWithoutOrderAndWithoutDuplicatesComparator implements Comparator> { @SuppressWarnings("SuspiciousMethodCalls") diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index feb12234f..db82686d7 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -19,12 +19,10 @@ import static org.mockito.Mockito.when; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -32,6 +30,7 @@ import org.junit.jupiter.api.Test; import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient; @@ -59,7 +58,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RecategorizationPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.queue.SearchTermOccurrencesResponseReceiver; import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionRequest; -import com.iqser.red.service.persistence.management.v1.processor.utils.DossierMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; @@ -80,10 +78,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary.Dictionary; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionBulkLocalRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel; @@ -4175,6 +4175,90 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { } } + @Test + public void testAddRedactionWithCommentToDossierAndTemplateDictionary() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + // Create a type that allows adding to dictionary and not to template dictionary + var type = typeProvider.testAndProvideType(dossierTemplate, dossier, "typeWithComment", true, 50); + assertThat(type.isDossierDictionaryOnly()).isTrue(); + + // Verify initial state of dictionaries + var initialDossierDictionary = internalDictionaryClient.getDictionaryForType( + toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); + assertThat(initialDossierDictionary.getEntries()).isEmpty(); + + var initialTemplateDictionary = internalDictionaryClient.getDictionaryForType( + toTypeId(type.getType(), dossierTemplate.getId()), null); + assertThat(initialTemplateDictionary.getEntries()).isEmpty(); + + // Add a manual redaction with a comment to the dossier dictionary + String comment = "This is a test comment for the redaction."; + String redactionValue = "Leia Organa"; + + ManualRedactionResponse addRedactionResponse = manualRedactionClient.addRedactionBulk( + dossier.getId(), + file.getId(), + Set.of(AddRedactionRequestModel.builder() + .positions(List.of(Rectangle.builder() + .topLeftX(100) + .topLeftY(200) + .width(50) + .height(20) + .page(1) + .build())) + .section("Confidential Section") + .addToDictionary(true) + .addToAllDossiers(false) + .type(type.getType()) + .reason("Confidential Information") + .value(redactionValue) + .legalBasis("GDPR") + .comment(new AddCommentRequestModel(comment)) + .build()) + ); + + // Extract the annotation ID from the response for verification + String annotationId = addRedactionResponse.getManualAnnotationResponses() + .stream() + .findFirst() + .map(ManualAnnotationResponse::getAnnotationId) + .orElseThrow(() -> new AssertionError("No annotation ID returned")); + + // Verify the redaction is present + var manualRedactions = manualRedactionClient.getManualRedactions( + dossier.getId(), file.getId(), false, true); + + assertThat(manualRedactions.getEntriesToAdd()).hasSize(1); + ManualRedactionEntry addedRedaction = manualRedactions.getEntriesToAdd() + .stream() + .filter(entry -> entry.getAnnotationId().equals(annotationId)) + .findFirst() + .orElseThrow(() -> new AssertionError("Redaction not found in manual redactions")); + + assertThat(addedRedaction.getValue()).isEqualTo(redactionValue); + + // Verify the comment was created + List comments = commentRepository.findAll(); + assertFalse(comments.isEmpty()); + assertEquals(comments.get(0).getText(), comment); + assertEquals(comments.get(0).getAnnotationId(), annotationId); + + // Verify the redaction is added to the dossier dictionary + var updatedDossierDictionary = internalDictionaryClient.getDictionaryForType( + toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); + assertThat(updatedDossierDictionary.getEntries().stream().map(DictionaryEntry::getValue)).containsExactly(redactionValue); + + // Verify the redaction was not added to the template dictionary + var updatedTemplateDictionary = internalDictionaryClient.getDictionaryForType( + toTypeId(type.getType(), dossierTemplate.getId()), null); + assertThat(updatedTemplateDictionary.getEntries()).isEmpty(); + } + + @Test public void testSingleForceAndBulkForceOnDictEntryAutomaticAnalysisOff() { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties b/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties index ead058aa4..ee9bd06b8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties @@ -1 +1 @@ -hub.image.name.prefix=docker-dev.knecon.com/tests/ +hub.image.name.prefix=docker-dev.knecon.com/tests/ \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntityLogLegalBasis.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntityLogLegalBasis.java index 15d26dfcf..39ca34ec6 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntityLogLegalBasis.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntityLogLegalBasis.java @@ -1,12 +1,14 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; @Data @NoArgsConstructor @AllArgsConstructor +@Builder public class EntityLogLegalBasis { private String name; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/imported/ImportedLegalBasis.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/imported/ImportedLegalBasis.java index 33668c71e..97da35a0f 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/imported/ImportedLegalBasis.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/imported/ImportedLegalBasis.java @@ -11,6 +11,7 @@ import lombok.NoArgsConstructor; @AllArgsConstructor public class ImportedLegalBasis { + private String technicalName; private String name; private String description; private String reason;