Merge branch 'RED-10715-4.3' into 'release/2.589.x'
RED-10715: Remove Saas migration See merge request redactmanager/persistence-service!917
This commit is contained in:
Dominique Eifländer
commit
e62cc3c53e
@ -1,111 +0,0 @@
|
|||||||
package com.iqser.red.persistence.service.v1.external.api.impl.controller;
|
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.external.resource.MigrationStatusResource;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
|
|
||||||
|
|
||||||
import lombok.AccessLevel;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.experimental.FieldDefaults;
|
|
||||||
|
|
||||||
import org.springframework.http.ResponseEntity;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*;
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class MigrationStatusController implements MigrationStatusResource {
|
|
||||||
|
|
||||||
SaasMigrationService saasMigrationService;
|
|
||||||
|
|
||||||
SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
|
|
||||||
|
|
||||||
FileStatusService fileStatusService;
|
|
||||||
|
|
||||||
|
|
||||||
public MigrationStatusResponse migrationStatus() {
|
|
||||||
|
|
||||||
int numberOfFilesToMigrate = saasMigrationStatusPersistenceService.countAll();
|
|
||||||
|
|
||||||
Map<SaasMigrationStatus, Integer> filesInStatus = new HashMap<>();
|
|
||||||
filesInStatus.put(MIGRATION_REQUIRED, saasMigrationStatusPersistenceService.countByStatus(MIGRATION_REQUIRED));
|
|
||||||
filesInStatus.put(DOCUMENT_FILES_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(DOCUMENT_FILES_MIGRATED));
|
|
||||||
filesInStatus.put(REDACTION_LOGS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(REDACTION_LOGS_MIGRATED));
|
|
||||||
filesInStatus.put(ANNOTATION_IDS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(ANNOTATION_IDS_MIGRATED));
|
|
||||||
filesInStatus.put(FINISHED, saasMigrationStatusPersistenceService.countByStatus(FINISHED));
|
|
||||||
filesInStatus.put(ERROR, saasMigrationStatusPersistenceService.countByStatus(ERROR));
|
|
||||||
|
|
||||||
var filesInErrorState = saasMigrationStatusPersistenceService.findAllByStatus(ERROR);
|
|
||||||
|
|
||||||
var errorCauses = filesInErrorState.stream()
|
|
||||||
.collect(Collectors.toMap(errorFile -> errorFile.getDossierId() + "/" + errorFile.getFileId(), SaasMigrationStatusEntity::getErrorCause));
|
|
||||||
|
|
||||||
return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ResponseEntity<?> startMigrationForFile(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
if (!fileStatusService.fileExists(fileId)) {
|
|
||||||
throw new NotFoundException(String.format("File with id %s does not exist", fileId));
|
|
||||||
}
|
|
||||||
|
|
||||||
saasMigrationService.startMigrationForFile(dossierId, fileId);
|
|
||||||
|
|
||||||
return ResponseEntity.ok().build();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ResponseEntity<?> revertMigrationForFile(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
if (!fileStatusService.fileExists(fileId)) {
|
|
||||||
throw new NotFoundException(String.format("File with id %s does not exist", fileId));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!saasMigrationStatusPersistenceService.findById(fileId).getStatus().equals(FINISHED)) {
|
|
||||||
throw new BadRequestException(String.format("File with id %s is not migrated yet, can't revert.", fileId));
|
|
||||||
}
|
|
||||||
|
|
||||||
saasMigrationService.revertMigrationForFile(dossierId, fileId);
|
|
||||||
|
|
||||||
return ResponseEntity.ok().build();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ResponseEntity<?> requeueErrorFiles() {
|
|
||||||
|
|
||||||
MigrationStatusResponse migrationStatus = migrationStatus();
|
|
||||||
if (!migrationIsFinished(migrationStatus)) {
|
|
||||||
throw new BadRequestException("There are still files processing, please wait until migration has finished to retry!");
|
|
||||||
}
|
|
||||||
|
|
||||||
saasMigrationService.requeueErrorFiles();
|
|
||||||
|
|
||||||
return ResponseEntity.ok().build();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static boolean migrationIsFinished(MigrationStatusResponse migrationStatus) {
|
|
||||||
|
|
||||||
return migrationStatus.getFilesInStatus().entrySet()
|
|
||||||
.stream()
|
|
||||||
.filter(e -> e.getValue() > 0)
|
|
||||||
.allMatch(e -> e.getKey().equals(FINISHED) || e.getKey().equals(ERROR));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,56 +0,0 @@
|
|||||||
package com.iqser.red.service.persistence.service.v1.api.external.resource;
|
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
|
|
||||||
|
|
||||||
import io.swagger.v3.oas.annotations.Operation;
|
|
||||||
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
|
||||||
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
|
||||||
|
|
||||||
import org.springframework.http.MediaType;
|
|
||||||
import org.springframework.http.ResponseEntity;
|
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
|
||||||
import org.springframework.web.bind.annotation.ResponseBody;
|
|
||||||
|
|
||||||
public interface MigrationStatusResource {
|
|
||||||
|
|
||||||
String MIGRATION_STATUS_REST_PATH = ExternalApi.BASE_PATH + "/migration-status";
|
|
||||||
String START_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/start_migration";
|
|
||||||
String REVERT_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/revert_migration";
|
|
||||||
String RETRY_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/retry_migration";
|
|
||||||
|
|
||||||
String FILE_ID = "fileId";
|
|
||||||
String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
|
|
||||||
|
|
||||||
String DOSSIER_ID = "dossierId";
|
|
||||||
String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
|
|
||||||
|
|
||||||
|
|
||||||
@ResponseBody
|
|
||||||
@PostMapping(value = MIGRATION_STATUS_REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
|
|
||||||
@Operation(summary = "Show the status of the migration", description = "None")
|
|
||||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
|
|
||||||
MigrationStatusResponse migrationStatus();
|
|
||||||
|
|
||||||
|
|
||||||
@ResponseBody
|
|
||||||
@PostMapping(value = START_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
|
|
||||||
@Operation(summary = "Start SAAS migration for specific file", description = "None")
|
|
||||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
|
|
||||||
ResponseEntity<?> startMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
|
|
||||||
|
|
||||||
|
|
||||||
@ResponseBody
|
|
||||||
@PostMapping(value = REVERT_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
|
|
||||||
@Operation(summary = "Start SAAS migration for specific file", description = "None")
|
|
||||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
|
|
||||||
ResponseEntity<?> revertMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
|
|
||||||
|
|
||||||
|
|
||||||
@ResponseBody
|
|
||||||
@PostMapping(value = RETRY_MIGRATION_REST_PATH)
|
|
||||||
@Operation(summary = "Restart SAAS migration for all files in error state", description = "None")
|
|
||||||
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
|
|
||||||
ResponseEntity<?> requeueErrorFiles();
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,35 +0,0 @@
|
|||||||
package com.iqser.red.service.persistence.management.v1.processor.entity.migration;
|
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
|
|
||||||
|
|
||||||
import jakarta.persistence.*;
|
|
||||||
import lombok.AllArgsConstructor;
|
|
||||||
import lombok.Builder;
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.NoArgsConstructor;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
@Builder
|
|
||||||
@AllArgsConstructor
|
|
||||||
@NoArgsConstructor
|
|
||||||
@Entity
|
|
||||||
@Table(name = "saas_migration_status")
|
|
||||||
public class SaasMigrationStatusEntity {
|
|
||||||
|
|
||||||
@Id
|
|
||||||
private String fileId;
|
|
||||||
|
|
||||||
@Column
|
|
||||||
private String dossierId;
|
|
||||||
|
|
||||||
@Column
|
|
||||||
@Enumerated(EnumType.STRING)
|
|
||||||
private SaasMigrationStatus status;
|
|
||||||
|
|
||||||
@Column
|
|
||||||
private Integer processingErrorCounter;
|
|
||||||
|
|
||||||
@Column
|
|
||||||
private String errorCause;
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,170 +0,0 @@
|
|||||||
package com.iqser.red.service.persistence.management.v1.processor.migration;
|
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
|
|
||||||
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
|
|
||||||
|
|
||||||
import jakarta.transaction.Transactional;
|
|
||||||
import lombok.AccessLevel;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.experimental.FieldDefaults;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
@Transactional
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
|
||||||
public class SaasAnnotationIdMigrationService {
|
|
||||||
|
|
||||||
ManualRedactionRepository manualRedactionRepository;
|
|
||||||
RemoveRedactionRepository removeRedactionRepository;
|
|
||||||
ForceRedactionRepository forceRedactionRepository;
|
|
||||||
ResizeRedactionRepository resizeRedactionRepository;
|
|
||||||
RecategorizationRepository recategorizationRepository;
|
|
||||||
LegalBasisChangeRepository legalBasisChangeRepository;
|
|
||||||
CommentRepository commentRepository;
|
|
||||||
FileRepository fileRepository;
|
|
||||||
|
|
||||||
|
|
||||||
public int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
|
||||||
|
|
||||||
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
var oldEntry = manualRedactionRepository.findById(oldAnnotationEntityId);
|
|
||||||
if (oldEntry.isPresent()) {
|
|
||||||
|
|
||||||
var newEntry = MagicConverter.convert(oldEntry.get(), ManualRedactionEntryEntity.class);
|
|
||||||
newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class));
|
|
||||||
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
|
|
||||||
.get());
|
|
||||||
newEntry.setId(newAnnotationEntityId);
|
|
||||||
|
|
||||||
manualRedactionRepository.deleteById(oldAnnotationEntityId);
|
|
||||||
manualRedactionRepository.save(newEntry);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
|
||||||
|
|
||||||
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
var oldEntry = removeRedactionRepository.findById(oldAnnotationEntityId);
|
|
||||||
if (oldEntry.isPresent()) {
|
|
||||||
|
|
||||||
var newEntry = MagicConverter.convert(oldEntry.get(), IdRemovalEntity.class);
|
|
||||||
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
|
|
||||||
.get());
|
|
||||||
newEntry.setId(newAnnotationEntityId);
|
|
||||||
|
|
||||||
removeRedactionRepository.deleteById(oldAnnotationEntityId);
|
|
||||||
removeRedactionRepository.save(newEntry);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
|
||||||
|
|
||||||
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
var oldEntry = forceRedactionRepository.findById(oldAnnotationEntityId);
|
|
||||||
if (oldEntry.isPresent()) {
|
|
||||||
|
|
||||||
var newEntry = MagicConverter.convert(oldEntry.get(), ManualForceRedactionEntity.class);
|
|
||||||
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
|
|
||||||
.get());
|
|
||||||
newEntry.setId(newAnnotationEntityId);
|
|
||||||
|
|
||||||
forceRedactionRepository.deleteById(oldAnnotationEntityId);
|
|
||||||
forceRedactionRepository.save(newEntry);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
|
||||||
|
|
||||||
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
var oldEntry = resizeRedactionRepository.findById(oldAnnotationEntityId);
|
|
||||||
if (oldEntry.isPresent()) {
|
|
||||||
|
|
||||||
var newEntry = MagicConverter.convert(oldEntry.get(), ManualResizeRedactionEntity.class);
|
|
||||||
newEntry.setId(newAnnotationEntityId);
|
|
||||||
newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class));
|
|
||||||
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
|
|
||||||
.get());
|
|
||||||
|
|
||||||
resizeRedactionRepository.deleteById(oldAnnotationEntityId);
|
|
||||||
resizeRedactionRepository.save(newEntry);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
|
||||||
|
|
||||||
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
var oldEntry = recategorizationRepository.findById(oldAnnotationEntityId);
|
|
||||||
if (oldEntry.isPresent()) {
|
|
||||||
|
|
||||||
var newEntry = MagicConverter.convert(oldEntry.get(), ManualRecategorizationEntity.class);
|
|
||||||
newEntry.setId(newAnnotationEntityId);
|
|
||||||
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
|
|
||||||
.get());
|
|
||||||
|
|
||||||
recategorizationRepository.deleteById(oldAnnotationEntityId);
|
|
||||||
recategorizationRepository.save(newEntry);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
|
||||||
|
|
||||||
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
var oldEntry = legalBasisChangeRepository.findById(oldAnnotationEntityId);
|
|
||||||
if (oldEntry.isPresent()) {
|
|
||||||
|
|
||||||
var newEntry = MagicConverter.convert(oldEntry.get(), ManualLegalBasisChangeEntity.class);
|
|
||||||
newEntry.setId(newAnnotationEntityId);
|
|
||||||
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
|
|
||||||
.get());
|
|
||||||
|
|
||||||
legalBasisChangeRepository.deleteById(oldAnnotationEntityId);
|
|
||||||
legalBasisChangeRepository.save(newEntry);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int updateCommentIds(String fileId, String key, String value) {
|
|
||||||
|
|
||||||
if (key.equals(value)) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,83 +0,0 @@
|
|||||||
package com.iqser.red.service.persistence.management.v1.processor.migration;
|
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashSet;
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import com.google.common.hash.HashFunction;
|
|
||||||
import com.google.common.hash.Hashing;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
|
||||||
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class SaasMigrationManualChangesUpdateService {
|
|
||||||
|
|
||||||
private final AddRedactionPersistenceService addRedactionPersistenceService;
|
|
||||||
|
|
||||||
private final HashFunction hashFunction = Hashing.murmur3_128();
|
|
||||||
|
|
||||||
|
|
||||||
public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId) {
|
|
||||||
|
|
||||||
var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly());
|
|
||||||
for (var unprocessedManualAdd : unprocessedManualAdds) {
|
|
||||||
|
|
||||||
if (!unprocessedManualAdd.getDictionaryEntryType().equals(DictionaryEntryType.ENTRY)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) {
|
|
||||||
// copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table.
|
|
||||||
// It has no functionality, its only there, such that the UI can show a pending change.
|
|
||||||
ManualRedactionEntryEntity pendingDictAdd = new ManualRedactionEntryEntity(buildSecondaryId(unprocessedManualAdd.getId(), fileId),
|
|
||||||
unprocessedManualAdd.getUser(),
|
|
||||||
unprocessedManualAdd.getTypeId(),
|
|
||||||
unprocessedManualAdd.getValue(),
|
|
||||||
unprocessedManualAdd.getReason(),
|
|
||||||
unprocessedManualAdd.getLegalBasis(),
|
|
||||||
unprocessedManualAdd.getSection(),
|
|
||||||
unprocessedManualAdd.isRectangle(),
|
|
||||||
unprocessedManualAdd.isAddToDictionary(),
|
|
||||||
unprocessedManualAdd.isAddToAllDossiers(),
|
|
||||||
unprocessedManualAdd.isAddToDossierDictionary(),
|
|
||||||
DictionaryEntryType.ENTRY,
|
|
||||||
unprocessedManualAdd.getRequestDate(),
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
new ArrayList<>(unprocessedManualAdd.getPositions()),
|
|
||||||
unprocessedManualAdd.getFileStatus(),
|
|
||||||
unprocessedManualAdd.getTextBefore(),
|
|
||||||
unprocessedManualAdd.getTextAfter(),
|
|
||||||
unprocessedManualAdd.getSourceId(),
|
|
||||||
new HashSet<>(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries()));
|
|
||||||
|
|
||||||
addRedactionPersistenceService.update(pendingDictAdd);
|
|
||||||
|
|
||||||
// change existing dict add to unprocessed manual add. ID must match with prior entry, such that other unprocessed manual changes may be applied to it.
|
|
||||||
unprocessedManualAdd.setAddToDictionary(false);
|
|
||||||
unprocessedManualAdd.setAddToAllDossiers(false);
|
|
||||||
unprocessedManualAdd.setLegalBasis("");
|
|
||||||
unprocessedManualAdd.setTypeIdsOfModifiedDictionaries(Collections.emptySet());
|
|
||||||
unprocessedManualAdd.setDictionaryEntryType(null);
|
|
||||||
|
|
||||||
addRedactionPersistenceService.update(unprocessedManualAdd);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private AnnotationEntityId buildSecondaryId(AnnotationEntityId annotationEntityId, String fileId) {
|
|
||||||
|
|
||||||
return new AnnotationEntityId(hashFunction.hashString(annotationEntityId.getAnnotationId(), StandardCharsets.UTF_8).toString(), fileId);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,398 +0,0 @@
|
|||||||
package com.iqser.red.service.persistence.management.v1.processor.migration;
|
|
||||||
|
|
||||||
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_REQUEST_QUEUE;
|
|
||||||
|
|
||||||
import java.time.OffsetDateTime;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
|
||||||
|
|
||||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
|
||||||
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
|
|
||||||
import com.iqser.red.storage.commons.exception.StorageException;
|
|
||||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
|
||||||
import com.iqser.red.storage.commons.service.StorageService;
|
|
||||||
import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService;
|
|
||||||
import com.knecon.fforesight.databasetenantcommons.providers.events.TenantSyncEvent;
|
|
||||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
|
|
||||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
|
||||||
import com.knecon.fforesight.tenantcommons.TenantProvider;
|
|
||||||
import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest;
|
|
||||||
|
|
||||||
import lombok.AccessLevel;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.experimental.FieldDefaults;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
|
||||||
public class SaasMigrationService implements TenantSyncService {
|
|
||||||
|
|
||||||
AutomaticAnalysisJob automaticAnalysisJob;
|
|
||||||
FileStatusPersistenceService fileStatusPersistenceService;
|
|
||||||
SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
|
|
||||||
DossierService dossierService;
|
|
||||||
ManualRedactionProviderService manualRedactionProviderService;
|
|
||||||
TenantProvider tenantProvider;
|
|
||||||
IndexingService indexingService;
|
|
||||||
LayoutParsingRequestFactory layoutParsingRequestFactory;
|
|
||||||
RabbitTemplate rabbitTemplate;
|
|
||||||
FileManagementServiceSettings settings;
|
|
||||||
StorageService storageService;
|
|
||||||
SaasAnnotationIdMigrationService saasAnnotationIdMigrationService;
|
|
||||||
UncompressedFilesMigrationService uncompressedFilesMigrationService;
|
|
||||||
ManualRedactionService manualRedactionService;
|
|
||||||
CommentService commentService;
|
|
||||||
RankDeDuplicationService rankDeDuplicationService;
|
|
||||||
SaasMigrationManualChangesUpdateService saasMigrationManualChangesUpdateService;
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public synchronized void syncTenant(TenantSyncEvent tenantSyncEvent) {
|
|
||||||
|
|
||||||
startMigrationForTenant(tenantSyncEvent.getTenantId());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Persistence-Service needs to be scaled to 1.
|
|
||||||
|
|
||||||
|
|
||||||
public void startMigrationForTenant(String tenantId) {
|
|
||||||
|
|
||||||
// TODO migrate rules.
|
|
||||||
automaticAnalysisJob.stopForTenant(tenantId);
|
|
||||||
|
|
||||||
log.info("Starting uncompressed files migration ...");
|
|
||||||
uncompressedFilesMigrationService.migrateUncompressedFiles(tenantId);
|
|
||||||
log.info("Finished uncompressed files migration ...");
|
|
||||||
|
|
||||||
rankDeDuplicationService.deduplicate();
|
|
||||||
int numberOfFiles = 0;
|
|
||||||
|
|
||||||
var files = saasMigrationStatusPersistenceService.findAll();
|
|
||||||
|
|
||||||
for (var file : files) {
|
|
||||||
|
|
||||||
var dossier = dossierService.getDossierById(file.getDossierId());
|
|
||||||
|
|
||||||
if (dossier.getHardDeletedTime() != null) {
|
|
||||||
if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
|
|
||||||
continue;
|
|
||||||
} else {
|
|
||||||
fileStatusPersistenceService.hardDelete(file.getFileId(), dossier.getHardDeletedTime());
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) {
|
|
||||||
log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// delete NER_ENTITIES since offsets depend on old document structure.
|
|
||||||
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(file.getDossierId(), file.getFileId(), FileType.NER_ENTITIES));
|
|
||||||
|
|
||||||
var layoutParsingRequest = layoutParsingRequestFactory.build(file.getDossierId(), file.getFileId(), false);
|
|
||||||
|
|
||||||
rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest);
|
|
||||||
|
|
||||||
numberOfFiles++;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId());
|
|
||||||
if (numberOfFiles == 0) {
|
|
||||||
finalizeMigration();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void startMigrationForFile(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
var dossier = dossierService.getDossierById(dossierId);
|
|
||||||
|
|
||||||
if (dossier.getHardDeletedTime() != null) {
|
|
||||||
if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
fileStatusPersistenceService.hardDelete(fileId, dossier.getHardDeletedTime());
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId);
|
|
||||||
saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
|
|
||||||
var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, false);
|
|
||||||
rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void handleLayoutParsingFinished(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
if (!layoutParsingFilesExist(dossierId, fileId)) {
|
|
||||||
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Layout parsing files not written!");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED);
|
|
||||||
|
|
||||||
if (fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
|
|
||||||
saasMigrationManualChangesUpdateService.convertUnprocessedAddToDictionariesToLocalChanges(fileId);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
indexingService.reindex(dossierId, Set.of(fileId), false);
|
|
||||||
|
|
||||||
String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId();
|
|
||||||
|
|
||||||
rabbitTemplate.convertAndSend(MIGRATION_REQUEST_QUEUE,
|
|
||||||
MigrationRequest.builder()
|
|
||||||
.dossierTemplateId(dossierTemplateId)
|
|
||||||
.dossierId(dossierId)
|
|
||||||
.fileId(fileId)
|
|
||||||
.fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED))
|
|
||||||
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
|
|
||||||
.entitiesWithComments(commentService.getCommentCounts(fileId).keySet())
|
|
||||||
.build());
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("Queuing of entityLog migration failed with {}", e.getMessage());
|
|
||||||
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage()));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private boolean layoutParsingFilesExist(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) //
|
|
||||||
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) //
|
|
||||||
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) //
|
|
||||||
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void handleEntityLogMigrationFinished(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
if (!entityLogMigrationFilesExist(dossierId, fileId)) {
|
|
||||||
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Migration Files not written!");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED);
|
|
||||||
|
|
||||||
log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
|
|
||||||
migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(dossierId, fileId);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private boolean entityLogMigrationFilesExist(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists(
|
|
||||||
TenantContext.getTenantId(),
|
|
||||||
StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void handleError(String dossierId, String fileId, String errorCause, String retryExchange) {
|
|
||||||
|
|
||||||
var migrationEntry = saasMigrationStatusPersistenceService.findById(fileId);
|
|
||||||
Integer numErrors = migrationEntry.getProcessingErrorCounter();
|
|
||||||
if (numErrors != null && numErrors <= settings.getMaxErrorRetries()) {
|
|
||||||
saasMigrationStatusPersistenceService.updateErrorCounter(fileId, numErrors + 1, errorCause);
|
|
||||||
rabbitTemplate.convertAndSend(retryExchange, TenantContext.getTenantId(), MigrationRequest.builder().dossierId(dossierId).fileId(fileId).build());
|
|
||||||
log.error("Retrying error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause);
|
|
||||||
} else {
|
|
||||||
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, errorCause);
|
|
||||||
log.error("Error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void requeueErrorFiles() {
|
|
||||||
|
|
||||||
automaticAnalysisJob.stopForTenant(TenantContext.getTenantId());
|
|
||||||
saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR)
|
|
||||||
.forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId()));
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
|
|
||||||
Map<String, String> oldToNewMapping = migratedIds.buildOldToNewMapping();
|
|
||||||
updateAnnotationIds(dossierId, fileId, oldToNewMapping);
|
|
||||||
List<String> forceRedactionIdsToDelete = migratedIds.getForceRedactionIdsToDelete();
|
|
||||||
softDeleteForceRedactions(fileId, forceRedactionIdsToDelete);
|
|
||||||
log.info("Soft-deleted force redactions.");
|
|
||||||
List<ManualRedactionEntry> manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd();
|
|
||||||
int count = addManualRedactionEntries(manualRedactionEntriesToAdd);
|
|
||||||
log.info("Added {} additional manual entries.", count);
|
|
||||||
deleteSectionGridAndNerEntitiesFiles(dossierId, fileId);
|
|
||||||
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
|
|
||||||
|
|
||||||
log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
|
|
||||||
finalizeMigration(); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void deleteSectionGridAndNerEntitiesFiles(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
try {
|
|
||||||
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
|
|
||||||
} catch (StorageObjectDoesNotExist e) {
|
|
||||||
log.info("No sectiongrid found for {}, {}, ignoring....", dossierId, fileId);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
|
|
||||||
} catch (StorageObjectDoesNotExist e) {
|
|
||||||
log.info("No ner entities file found for {}, {}, ignoring....", dossierId, fileId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void softDeleteForceRedactions(String fileId, List<String> forceRedactionIdsToDelete) {
|
|
||||||
|
|
||||||
manualRedactionService.softDeleteForceRedactions(fileId, forceRedactionIdsToDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private int addManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
|
|
||||||
|
|
||||||
manualRedactionEntriesToAdd.forEach(add -> {
|
|
||||||
if (add.getSection() != null && add.getSection().length() > 254) {
|
|
||||||
add.setSection(add.getSection().substring(0, 254));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void revertMigrationForFile(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
log.info("Reverting Migration for dossierId {} and fileId {}", dossierId, fileId);
|
|
||||||
MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
|
|
||||||
Map<String, String> newToOldMapping = migratedIds.buildNewToOldMapping();
|
|
||||||
updateAnnotationIds(dossierId, fileId, newToOldMapping);
|
|
||||||
deleteManualRedactionEntries(migratedIds.getManualRedactionEntriesToAdd());
|
|
||||||
undeleteForceRedactions(fileId, migratedIds.getForceRedactionIdsToDelete());
|
|
||||||
saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void undeleteForceRedactions(String fileId, List<String> forceRedactionIdsToDelete) {
|
|
||||||
|
|
||||||
manualRedactionService.undeleteForceRedactions(fileId, forceRedactionIdsToDelete);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void deleteManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
|
|
||||||
|
|
||||||
manualRedactionService.deleteManualRedactionEntries(manualRedactionEntriesToAdd);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void updateAnnotationIds(String dossierId, String fileId, Map<String, String> idMapping) {
|
|
||||||
|
|
||||||
try {
|
|
||||||
updateAnnotationIds(fileId, idMapping);
|
|
||||||
} catch (Exception e) {
|
|
||||||
String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s",
|
|
||||||
TenantContext.getTenantId(),
|
|
||||||
dossierId,
|
|
||||||
fileId,
|
|
||||||
e.getMessage());
|
|
||||||
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message);
|
|
||||||
log.error(message);
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void finalizeMigration() {
|
|
||||||
|
|
||||||
if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
|
|
||||||
// automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
|
|
||||||
tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build());
|
|
||||||
log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void updateAnnotationIds(String fileId, Map<String, String> idMapping) {
|
|
||||||
|
|
||||||
AtomicInteger numUpdates = new AtomicInteger(0);
|
|
||||||
AtomicInteger numCommentUpdates = new AtomicInteger(0);
|
|
||||||
idMapping.forEach((key, value) -> {
|
|
||||||
AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key);
|
|
||||||
AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value);
|
|
||||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
|
||||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
|
||||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
|
||||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
|
||||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
|
||||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
|
||||||
numCommentUpdates.getAndAdd(saasAnnotationIdMigrationService.updateCommentIds(fileId, key, value));
|
|
||||||
});
|
|
||||||
log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
|
|
||||||
|
|
||||||
return new AnnotationEntityId(annotationId, fileId);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private MigratedIds getMigratedIds(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
try {
|
|
||||||
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS), MigratedIds.class);
|
|
||||||
} catch (StorageObjectDoesNotExist e) {
|
|
||||||
throw new NotFoundException(String.format("MigratedIds does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId));
|
|
||||||
} catch (StorageException e) {
|
|
||||||
throw new InternalServerErrorException(e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -14,7 +14,6 @@ import org.springframework.stereotype.Service;
|
|||||||
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
|
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
|
import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||||
@ -38,7 +37,6 @@ public class AutomaticAnalysisJob implements Job {
|
|||||||
private final FileStatusService fileStatusService;
|
private final FileStatusService fileStatusService;
|
||||||
private final TenantProvider tenantProvider;
|
private final TenantProvider tenantProvider;
|
||||||
private final ObservationRegistry observationRegistry;
|
private final ObservationRegistry observationRegistry;
|
||||||
private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
|
|
||||||
|
|
||||||
@Setter
|
@Setter
|
||||||
private boolean schedulingStopped;
|
private boolean schedulingStopped;
|
||||||
@ -69,11 +67,6 @@ public class AutomaticAnalysisJob implements Job {
|
|||||||
|
|
||||||
TenantContext.setTenantId(tenant.getTenantId());
|
TenantContext.setTenantId(tenant.getTenantId());
|
||||||
|
|
||||||
if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) {
|
|
||||||
log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
String queueName = MessagingConfiguration.REDACTION_REQUEST_QUEUE_PREFIX + "_" + tenant.getTenantId();
|
String queueName = MessagingConfiguration.REDACTION_REQUEST_QUEUE_PREFIX + "_" + tenant.getTenantId();
|
||||||
var redactionQueueInfo = amqpAdmin.getQueueInfo(queueName);
|
var redactionQueueInfo = amqpAdmin.getQueueInfo(queueName);
|
||||||
if (redactionQueueInfo != null) {
|
if (redactionQueueInfo != null) {
|
||||||
|
|||||||
@ -1,96 +0,0 @@
|
|||||||
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
|
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
|
|
||||||
|
|
||||||
import jakarta.transaction.Transactional;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class SaasMigrationStatusPersistenceService {
|
|
||||||
|
|
||||||
private final SaasMigrationStatusRepository saasMigrationStatusRepository;
|
|
||||||
|
|
||||||
|
|
||||||
public List<SaasMigrationStatusEntity> findAllByStatus(SaasMigrationStatus status) {
|
|
||||||
|
|
||||||
return saasMigrationStatusRepository.findAllByStatus(status);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public SaasMigrationStatusEntity findById(String fileId) {
|
|
||||||
|
|
||||||
var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId);
|
|
||||||
if (migrationStatusOptional.isPresent()) {
|
|
||||||
return migrationStatusOptional.get();
|
|
||||||
}
|
|
||||||
throw new NotFoundException("No migration entry found for fileId" + fileId);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public boolean isMigrating(String fileId) {
|
|
||||||
|
|
||||||
var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId);
|
|
||||||
return migrationStatusOptional.isPresent() && migrationStatusOptional.get().getStatus() != SaasMigrationStatus.FINISHED;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean migrationFinishedForTenant() {
|
|
||||||
|
|
||||||
return saasMigrationStatusRepository.findAllWhereStatusNotFinishedAndNotError() == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Transactional
|
|
||||||
public void createMigrationRequiredStatus(String dossierId, String fileId) {
|
|
||||||
|
|
||||||
saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder().fileId(fileId).dossierId(dossierId).status(SaasMigrationStatus.MIGRATION_REQUIRED).build());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Transactional
|
|
||||||
public void updateStatus(String fileId, SaasMigrationStatus status) {
|
|
||||||
|
|
||||||
saasMigrationStatusRepository.updateStatus(fileId, status);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Transactional
|
|
||||||
public void updateErrorStatus(String fileId, String errorCause) {
|
|
||||||
|
|
||||||
saasMigrationStatusRepository.updateErrorStatus(fileId, SaasMigrationStatus.ERROR, errorCause);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Transactional
|
|
||||||
public void updateErrorCounter(String fileId, Integer processingErrorCounter, String errorCause) {
|
|
||||||
|
|
||||||
saasMigrationStatusRepository.updateErrorCounter(fileId, processingErrorCounter, errorCause);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int countByStatus(SaasMigrationStatus status) {
|
|
||||||
|
|
||||||
return saasMigrationStatusRepository.countByStatus(status);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int countAll() {
|
|
||||||
|
|
||||||
return saasMigrationStatusRepository.countAll();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public List<SaasMigrationStatusEntity> findAll() {
|
|
||||||
|
|
||||||
return saasMigrationStatusRepository.findAll();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,44 +0,0 @@
|
|||||||
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
|
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
|
|
||||||
|
|
||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
|
||||||
import org.springframework.data.jpa.repository.Modifying;
|
|
||||||
import org.springframework.data.jpa.repository.Query;
|
|
||||||
import org.springframework.data.repository.query.Param;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public interface SaasMigrationStatusRepository extends JpaRepository<SaasMigrationStatusEntity, String> {
|
|
||||||
|
|
||||||
List<SaasMigrationStatusEntity> findAllByStatus(SaasMigrationStatus status);
|
|
||||||
|
|
||||||
|
|
||||||
@Modifying
|
|
||||||
@Query("update SaasMigrationStatusEntity e set e.status = :status where e.fileId = :fileId")
|
|
||||||
void updateStatus(@Param("fileId") String fileId, @Param("status") SaasMigrationStatus status);
|
|
||||||
|
|
||||||
|
|
||||||
@Modifying
|
|
||||||
@Query("update SaasMigrationStatusEntity e set e.status = :status, e.errorCause = :errorCause where e.fileId = :fileId")
|
|
||||||
void updateErrorStatus(@Param("fileId") String fileId, @Param("status") SaasMigrationStatus status, @Param("errorCause") String errorCause);
|
|
||||||
|
|
||||||
|
|
||||||
@Modifying
|
|
||||||
@Query("update SaasMigrationStatusEntity e set e.processingErrorCounter = :processingErrorCounter, e.errorCause = :errorCause where e.fileId = :fileId")
|
|
||||||
void updateErrorCounter(@Param("fileId") String fileId, @Param("processingErrorCounter") Integer processingErrorCounter, @Param("errorCause") String errorCause);
|
|
||||||
|
|
||||||
|
|
||||||
@Query("select count(*) from SaasMigrationStatusEntity e where e.status = :status")
|
|
||||||
int countByStatus(@Param("status") SaasMigrationStatus status);
|
|
||||||
|
|
||||||
|
|
||||||
@Query("select count(*) from SaasMigrationStatusEntity")
|
|
||||||
int countAll();
|
|
||||||
|
|
||||||
|
|
||||||
@Query("select count(*) from SaasMigrationStatusEntity e where e.status != 'FINISHED' and e.status != 'ERROR'")
|
|
||||||
int findAllWhereStatusNotFinishedAndNotError();
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -11,14 +11,12 @@ import org.springframework.stereotype.Service;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
|
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.ImageSimilarityService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.ImageSimilarityService;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.QueueMessageIdentifierService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.QueueMessageIdentifierService;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||||
@ -40,8 +38,6 @@ public class LayoutParsingFinishedMessageReceiver {
|
|||||||
private final FileStatusService fileStatusService;
|
private final FileStatusService fileStatusService;
|
||||||
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
|
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
|
|
||||||
private final SaasMigrationService saasMigrationService;
|
|
||||||
private final ImageSimilarityService imageSimilarityService;
|
private final ImageSimilarityService imageSimilarityService;
|
||||||
private final WebsocketService websocketService;
|
private final WebsocketService websocketService;
|
||||||
|
|
||||||
@ -53,11 +49,7 @@ public class LayoutParsingFinishedMessageReceiver {
|
|||||||
var dossierId = QueueMessageIdentifierService.parseDossierId(response.identifier());
|
var dossierId = QueueMessageIdentifierService.parseDossierId(response.identifier());
|
||||||
var fileId = QueueMessageIdentifierService.parseFileId(response.identifier());
|
var fileId = QueueMessageIdentifierService.parseFileId(response.identifier());
|
||||||
log.info("Layout parsing has finished for {}/{} in {}", dossierId, fileId, LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE);
|
log.info("Layout parsing has finished for {}/{} in {}", dossierId, fileId, LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE);
|
||||||
if (saasMigrationStatusPersistenceService.isMigrating(QueueMessageIdentifierService.parseFileId(response.identifier()))) {
|
|
||||||
saasMigrationService.handleLayoutParsingFinished(QueueMessageIdentifierService.parseDossierId(response.identifier()),
|
|
||||||
QueueMessageIdentifierService.parseFileId(response.identifier()));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var templateId = "";
|
var templateId = "";
|
||||||
var storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
|
var storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
|
||||||
@ -90,13 +82,6 @@ public class LayoutParsingFinishedMessageReceiver {
|
|||||||
if (errorCause == null) {
|
if (errorCause == null) {
|
||||||
errorCause = "Error occured during layout parsing!";
|
errorCause = "Error occured during layout parsing!";
|
||||||
}
|
}
|
||||||
if (saasMigrationStatusPersistenceService.isMigrating(QueueMessageIdentifierService.parseFileId(analyzeRequest.identifier()))) {
|
|
||||||
saasMigrationService.handleError(QueueMessageIdentifierService.parseDossierId(analyzeRequest.identifier()),
|
|
||||||
QueueMessageIdentifierService.parseFileId(analyzeRequest.identifier()),
|
|
||||||
errorCause,
|
|
||||||
LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER);
|
OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER);
|
||||||
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
|
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
|
||||||
|
|||||||
@ -1,53 +0,0 @@
|
|||||||
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
|
|
||||||
|
|
||||||
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_DLQ;
|
|
||||||
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_REQUEST_QUEUE;
|
|
||||||
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_RESPONSE_QUEUE;
|
|
||||||
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
|
|
||||||
|
|
||||||
import org.springframework.amqp.core.Message;
|
|
||||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
|
|
||||||
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
|
|
||||||
import com.iqser.red.service.redaction.v1.model.MigrationResponse;
|
|
||||||
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.SneakyThrows;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class RedactionServiceSaasMigrationMessageReceiver {
|
|
||||||
|
|
||||||
private final SaasMigrationService saasMigrationService;
|
|
||||||
private final ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
@RabbitListener(queues = MIGRATION_RESPONSE_QUEUE)
|
|
||||||
public void receive(MigrationResponse response) {
|
|
||||||
|
|
||||||
saasMigrationService.handleEntityLogMigrationFinished(response.getDossierId(), response.getFileId());
|
|
||||||
|
|
||||||
log.info("Received message {} in {}", response, MIGRATION_RESPONSE_QUEUE);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
@RabbitListener(queues = MIGRATION_DLQ)
|
|
||||||
public void handleDLQMessage(Message failedMessage) {
|
|
||||||
|
|
||||||
var migrationRequest = objectMapper.readValue(failedMessage.getBody(), MigrationRequest.class);
|
|
||||||
String errorCause = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
|
|
||||||
if (errorCause == null) {
|
|
||||||
errorCause = "Error occured during entityLog migration!";
|
|
||||||
}
|
|
||||||
saasMigrationService.handleError(migrationRequest.getDossierId(), migrationRequest.getFileId(), errorCause, MIGRATION_REQUEST_QUEUE);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
databaseChangeLog:
|
||||||
|
- changeSet:
|
||||||
|
id: drop-saas_migration_status-if-exists
|
||||||
|
author: dom
|
||||||
|
comment: drop saas_migration_status if exists
|
||||||
|
preConditions:
|
||||||
|
- onFail: MARK_RAN
|
||||||
|
- tableExists:
|
||||||
|
tableName: saas_migration_status
|
||||||
|
changes:
|
||||||
|
- dropTable:
|
||||||
|
tableName: saas_migration_status
|
||||||
@ -372,7 +372,8 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest {
|
|||||||
IOUtils.toByteArray(new ClassPathResource("files/csv/fileattributes_missing_quotation_mark.csv").getInputStream()));
|
IOUtils.toByteArray(new ClassPathResource("files/csv/fileattributes_missing_quotation_mark.csv").getInputStream()));
|
||||||
|
|
||||||
result = assertThrows(FeignException.class, () -> uploadClient.upload(missingQuotation, dossier.getId(), false, false));
|
result = assertThrows(FeignException.class, () -> uploadClient.upload(missingQuotation, dossier.getId(), false, false));
|
||||||
assertTrue(result.getMessage().contains("Invalid CSV file format: Unterminated quoted field at end of CSV line. Beginning of lost text: [4.636.0,4.363.0,4.363.0\\n]"));
|
assertTrue(result.getMessage().contains("Invalid CSV file format: Unterminated quoted field at end of CSV line. Beginning of lost text: [4.636.0,4.363.0,4.363.0\\n]")
|
||||||
|
|| result.getMessage().contains("Invalid CSV file format: Unterminiertes Anführungszeichen am Ende einer CSV-Zeile. Anfang des verlorenen Textes: [4.636.0,4.363.0,4.363.0\\n]"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user