Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-3800 Migration cleanup

Browse Source
This commit is contained in:
Timo Bejan 2022-04-29 13:29:17 +03:00
parent bb2157f4e1
commit 874c94d702
7 changed files with 30 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java
View File

@ -1,10 +1,5 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations; package com.iqser.red.service.peristence.v1.server.migration.migrations;
import java.util.ArrayList;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.migration.Migration; import com.iqser.red.service.peristence.v1.server.migration.Migration;
@ -13,11 +8,13 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import lombok.Setter; import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@Slf4j @Slf4j
@Setter @Setter
@ -56,8 +53,11 @@ public class DictionaryToEntityMigration2 extends Migration {
files.forEach(file -> { files.forEach(file -> {
log.info("Start migration of file {}", file.getId()); log.info("Start migration of file {}", file.getId());
if (file.getHardDeletedTime() == null) { if (file.getHardDeletedTime() == null) {
var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
try { try {
var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId()); var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId());
redactionLog.getRedactionLogEntry().forEach(entry -> { redactionLog.getRedactionLogEntry().forEach(entry -> {
@ -69,7 +69,7 @@ public class DictionaryToEntityMigration2 extends Migration {
if (entry.getType().startsWith("recommendation_")) { if (entry.getType().startsWith("recommendation_")) {
entry.setType(entry.getType().substring(15)); entry.setType(entry.getType().substring(15));
entry.setRecommendation(true); entry.setRecommendation(true);
log.info("removed _recommendation"); log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId());
} }
newRedactionLogEntries.add(entry); newRedactionLogEntries.add(entry);
@ -77,12 +77,11 @@ public class DictionaryToEntityMigration2 extends Migration {
redactionLog.setRedactionLogEntry(newRedactionLogEntries); redactionLog.setRedactionLogEntry(newRedactionLogEntries);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
log.info("Stored dossierId: {} and fileId: {}", dossier.getId(), file.getId()); log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
} catch (JsonProcessingException e) { } catch (JsonProcessingException e) {
throw new RuntimeException("Migration failed"); throw new RuntimeException("Migration failed");
} catch (NotFoundException e) { } catch (NotFoundException e) {
log.info("redactionLog {} does not exsist", file.getId()); log.info("RedactionLog does not exist for file with id {}.", file.getId());
return;
} }
} }
}); });

10
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java
View File

@ -48,9 +48,13 @@ public class MigrateHighlights3 extends Migration {
files.forEach(file -> { files.forEach(file -> {
if (file.getHardDeletedTime() == null) { if (file.getHardDeletedTime() == null) {
var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN); if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) {
var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes)); var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN);
fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights); var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes));
fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights);
} else {
log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId());
}
} }
}); });
}); });

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java
View File

@ -59,6 +59,9 @@ public class RemoveFalsePositiveManualRedactions6 extends Migration {
} }
}); });
} }
log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(),annotationIdsToRemove);
if (!annotationIdsToRemove.isEmpty()) { if (!annotationIdsToRemove.isEmpty()) {
annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id)); annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id));
} }

1
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java
View File

@ -65,6 +65,7 @@ public class TypeToEntityMigration5 extends Migration {
.filter(t -> t.getType().equals("false_positive")) .filter(t -> t.getType().equals("false_positive"))
.findFirst(); .findFirst();
if (falsePositive.isEmpty()) { if (falsePositive.isEmpty()) {
log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName());
return; return;
} }
typeIdsToDelete.add(falsePositive.get().getId()); typeIdsToDelete.add(falsePositive.get().getId());

19
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java
View File

@ -101,22 +101,8 @@ public class FileManagementStorageService {
} }
} }
public boolean objectExists(String dossierId, String fileId, FileType origin) {
public boolean imageInfoExists(String dossierId, String fileId) { return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, origin));
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
}
public boolean textExists(String dossierId, String fileId) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
}
public boolean nerEntitiesExists(String dossierId, String fileId) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
} }
@ -131,4 +117,5 @@ public class FileManagementStorageService {
storageService.deleteObject(storageId); storageService.deleteObject(storageId);
} }
} }

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.peristence.v1.server.service; package com.iqser.red.service.peristence.v1.server.service;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import org.springframework.retry.support.RetryTemplate; import org.springframework.retry.support.RetryTemplate;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
@ -34,7 +35,7 @@ public class FileStatusProcessingUpdateService {
switch (analyzeResult.getMessageType()) { switch (analyzeResult.getMessageType()) {
case STRUCTURE_ANALYSE: case STRUCTURE_ANALYSE:
if (settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) { if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
fileStatusService.addToNerQueue(dossierId, fileId); fileStatusService.addToNerQueue(dossierId, fileId);
} else { } else {

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -222,7 +222,7 @@ public class FileStatusService {
boolean reanalyse = isReanalyse(dossier, fileStatus); boolean reanalyse = isReanalyse(dossier, fileStatus);
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) { if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
addToImageQueue(dossierId, fileId); addToImageQueue(dossierId, fileId);
return; return;
@ -230,7 +230,7 @@ public class FileStatusService {
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus()); MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus());
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) { if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
addToNerQueue(dossierId, fileId); addToNerQueue(dossierId, fileId);
return; return;
@ -282,7 +282,7 @@ public class FileStatusService {
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
ProcessingStatus processingStatus) { ProcessingStatus processingStatus) {
if (!fileManagementStorageService.textExists(dossierId, fileId)) { if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
return MessageType.STRUCTURE_ANALYSE; return MessageType.STRUCTURE_ANALYSE;
} }
if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) { if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {