Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-3800 Migration cleanup

Browse Source
This commit is contained in:
Timo Bejan 2022-04-29 13:29:17 +03:00
parent bb2157f4e1
commit 874c94d702
7 changed files with 30 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java
View File

@ -1,10 +1,5 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import java.util.ArrayList;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
@ -13,11 +8,13 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@Slf4j
@Setter
@ -56,8 +53,11 @@ public class DictionaryToEntityMigration2 extends Migration {
files.forEach(file -> {
log.info("Start migration of file {}", file.getId());
if (file.getHardDeletedTime() == null) {
var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
try {
var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId());
redactionLog.getRedactionLogEntry().forEach(entry -> {
@ -69,7 +69,7 @@ public class DictionaryToEntityMigration2 extends Migration {
if (entry.getType().startsWith("recommendation_")) {
entry.setType(entry.getType().substring(15));
entry.setRecommendation(true);
log.info("removed _recommendation");
log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId());
}
newRedactionLogEntries.add(entry);
@ -77,12 +77,11 @@ public class DictionaryToEntityMigration2 extends Migration {
redactionLog.setRedactionLogEntry(newRedactionLogEntries);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
log.info("Stored dossierId: {} and fileId: {}", dossier.getId(), file.getId());
log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
} catch (JsonProcessingException e) {
throw new RuntimeException("Migration failed");
} catch (NotFoundException e) {
log.info("redactionLog {} does not exsist", file.getId());
return;
log.info("RedactionLog does not exist for file with id {}.", file.getId());
}
}
});

10
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java
View File

@ -48,9 +48,13 @@ public class MigrateHighlights3 extends Migration {
files.forEach(file -> {
if (file.getHardDeletedTime() == null) {
var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN);
var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes));
fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights);
if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) {
var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN);
var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes));
fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights);
} else {
log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId());
}
}
});
});

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java
View File

@ -59,6 +59,9 @@ public class RemoveFalsePositiveManualRedactions6 extends Migration {
}
});
}
log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(),annotationIdsToRemove);
if (!annotationIdsToRemove.isEmpty()) {
annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id));
}

1
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java
View File

@ -65,6 +65,7 @@ public class TypeToEntityMigration5 extends Migration {
.filter(t -> t.getType().equals("false_positive"))
.findFirst();
if (falsePositive.isEmpty()) {
log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName());
return;
}
typeIdsToDelete.add(falsePositive.get().getId());

19
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java
View File

@ -101,22 +101,8 @@ public class FileManagementStorageService {
}
}
public boolean imageInfoExists(String dossierId, String fileId) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
}
public boolean textExists(String dossierId, String fileId) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
}
public boolean nerEntitiesExists(String dossierId, String fileId) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
public boolean objectExists(String dossierId, String fileId, FileType origin) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, origin));
}
@ -131,4 +117,5 @@ public class FileManagementStorageService {
storageService.deleteObject(storageId);
}
}

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.web.bind.annotation.RestController;
@ -34,7 +35,7 @@ public class FileStatusProcessingUpdateService {
switch (analyzeResult.getMessageType()) {
case STRUCTURE_ANALYSE:
if (settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) {
if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
fileStatusService.addToNerQueue(dossierId, fileId);
} else {

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -222,7 +222,7 @@ public class FileStatusService {
boolean reanalyse = isReanalyse(dossier, fileStatus);
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) {
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
addToImageQueue(dossierId, fileId);
return;
@ -230,7 +230,7 @@ public class FileStatusService {
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus());
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) {
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
addToNerQueue(dossierId, fileId);
return;
@ -282,7 +282,7 @@ public class FileStatusService {
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
ProcessingStatus processingStatus) {
if (!fileManagementStorageService.textExists(dossierId, fileId)) {
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
return MessageType.STRUCTURE_ANALYSE;
}
if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {