RED-7763 - Use EntityLog in tests in persistence-service
- remove unneeded migrations
This commit is contained in:
Corina Olariu
parent
8d0cafd0c8
commit
12cdc83ee3
@ -1,73 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class DeleteRemovedManualAddRedactions7 extends Migration {
|
||||
|
||||
private static final String NAME = "Delete removed ManualAddRedactions";
|
||||
private static final long VERSION = 7;
|
||||
|
||||
@Autowired
|
||||
private AddRedactionPersistenceService addRedactionPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private RemoveRedactionPersistenceService removeRedactionPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private ManualRedactionProviderService manualRedactionProviderService;
|
||||
|
||||
|
||||
public DeleteRemovedManualAddRedactions7() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
List<ManualRedactionEntryEntity> addRedactions = addRedactionPersistenceService.findAllAddRedactions();
|
||||
|
||||
for (IdRemovalEntity idRemovalEntity : removeRedactionPersistenceService.findAllRemoveRedactions()) {
|
||||
if (manualAddRedactionsContains(addRedactions, idRemovalEntity.getId().getAnnotationId())) {
|
||||
log.info("hard delete ManualRedactions for file {} and annotation {}", idRemovalEntity.getId().getFileId(), idRemovalEntity.getId().getAnnotationId());
|
||||
manualRedactionProviderService.hardDeleteManualRedactions(idRemovalEntity.getId().getFileId(), idRemovalEntity.getId().getAnnotationId());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private boolean manualAddRedactionsContains(List<ManualRedactionEntryEntity> addRedactions, String annotationId) {
|
||||
|
||||
for (ManualRedactionEntryEntity manualRedactionEntryEntity : addRedactions) {
|
||||
if (StringUtils.equals(annotationId, manualRedactionEntryEntity.getId().getAnnotationId())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,89 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class DictionaryToEntityMigration2 extends Migration {
|
||||
|
||||
private static final String NAME = "Migrate RedactionLogs remove false_positive and recommendation_ prefix";
|
||||
private static final long VERSION = 2;
|
||||
|
||||
@Autowired
|
||||
private DossierPersistenceService dossierPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private FileStatusPersistenceService fileStatusPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
|
||||
public DictionaryToEntityMigration2() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
var dossiers = dossierPersistenceService.findAllDossiers();
|
||||
dossiers.forEach(dossier -> {
|
||||
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId());
|
||||
log.info("Start migration of dossier {}", dossier.getId());
|
||||
files.forEach(file -> {
|
||||
log.info("Start migration of file {}", file.getId());
|
||||
if (file.getHardDeletedTime() == null) {
|
||||
|
||||
try {
|
||||
var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
|
||||
var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId());
|
||||
redactionLog.getRedactionLogEntry().forEach(entry -> {
|
||||
|
||||
if (entry.getType().equals("false_positive")) {
|
||||
log.info("skipping false_positive for dossier {} and file {}", dossier.getId(), file.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
if (entry.getType().startsWith("recommendation_")) {
|
||||
entry.setType(entry.getType().substring(15));
|
||||
entry.setRecommendation(true);
|
||||
log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId());
|
||||
}
|
||||
|
||||
newRedactionLogEntries.add(entry);
|
||||
});
|
||||
|
||||
if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) {
|
||||
redactionLog.setRedactionLogEntry(newRedactionLogEntries);
|
||||
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
|
||||
log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.info("redactionLog {} does not exsist", file.getId());
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,74 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class EntityTypesMigration4 extends Migration {
|
||||
|
||||
private static final String NAME = "Set flags systemManaged, autoHideSkipped and hasDictionary for all entity types";
|
||||
private static final long VERSION = 4;
|
||||
|
||||
@Autowired
|
||||
private DictionaryPersistenceService dictionaryPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private EntryPersistenceService entryPersistenceService;
|
||||
|
||||
|
||||
public EntityTypesMigration4() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
Set<String> systemManagedTypes = Set.of("ocr", "image", "logo", "signature", "formula", "imported_redaction", "dossier_redaction");
|
||||
Set<String> autoHideSkippedTypes = Set.of("imported_redaction");
|
||||
|
||||
log.info("Will check all type entities and set systemManaged flag true for {}", systemManagedTypes);
|
||||
log.info("Will check all type entities and set autoHideSkipped flag true for {}", autoHideSkippedTypes);
|
||||
log.info("Will check all type entities and set hasDictionary flag true for types with entities");
|
||||
|
||||
List<TypeEntity> types = dictionaryPersistenceService.getAllTypes(false);
|
||||
|
||||
for (TypeEntity type : types) {
|
||||
if (type != null) {
|
||||
|
||||
// Check if type is systemManaged
|
||||
type.setSystemManaged(type.getType() != null && systemManagedTypes.contains(type.getType()));
|
||||
|
||||
// Check if type is autoHideSkipped
|
||||
type.setAutoHideSkipped(type.getType() != null && autoHideSkippedTypes.contains(type.getType()));
|
||||
|
||||
// Check if type has dictionaries
|
||||
List<? extends BaseDictionaryEntry> entries = entryPersistenceService.getEntries(type.getId(), DictionaryEntryType.ENTRY, null);
|
||||
|
||||
type.setHasDictionary(entries != null && !entries.isEmpty());
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
log.info("Save all types");
|
||||
dictionaryPersistenceService.saveAllTypes(types);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,66 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class
|
||||
|
||||
FileSizeMigration8 extends Migration {
|
||||
|
||||
private static final String NAME = "Update file size";
|
||||
private static final long VERSION = 8;
|
||||
|
||||
@Autowired
|
||||
private FileStatusPersistenceService fileStatusPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
|
||||
public FileSizeMigration8() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
List<FileEntity> allFiles = fileStatusPersistenceService.getAllFiles();
|
||||
|
||||
log.info("Number of all files: {}", allFiles.size());
|
||||
|
||||
allFiles.forEach(file -> {
|
||||
// not hard deleted
|
||||
if (file.getHardDeletedTime() == null) {
|
||||
try {
|
||||
var originFile = fileManagementStorageService.getStoredObjectBytes(file.getDossierId(), file.getId(), FileType.ORIGIN);
|
||||
fileStatusPersistenceService.updateFileSize(file.getId(), originFile.length);
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to load file bytes for file: {} in dossier {}. Error message: {} ", file.getId(), file.getDossierId(), e.getMessage());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,36 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class IndexMigration1 extends Migration {
|
||||
|
||||
private static final String NAME = "Recreate index and index all files";
|
||||
private static final long VERSION = 1;
|
||||
@Autowired
|
||||
private IndexingService indexingService;
|
||||
|
||||
|
||||
public IndexMigration1() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
log.info("Will call SearchService via queue to close, drop, recreate index and reindex all files");
|
||||
indexingService.reindex(null, null, true);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class ManualRedactionTypeMigration9 extends Migration {
|
||||
|
||||
private static final String NAME = "Add Manual Redaction Type";
|
||||
private static final long VERSION = 9;
|
||||
private static final String MANUAL_TYPE = "manual";
|
||||
|
||||
@Autowired
|
||||
private DictionaryPersistenceService dictionaryPersistenceService;
|
||||
@Autowired
|
||||
private DossierTemplateRepository dossierTemplateRepository;
|
||||
@Autowired
|
||||
private TypeRepository typeRepository;
|
||||
|
||||
|
||||
public ManualRedactionTypeMigration9() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
var allDossierTemplates = dossierTemplateRepository.findAll();
|
||||
|
||||
allDossierTemplates.forEach(dossierTemplateEntity -> {
|
||||
|
||||
var entitiesForDossierTemplate = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateEntity.getId(), true);
|
||||
|
||||
var manualType = entitiesForDossierTemplate.stream().filter(e -> MANUAL_TYPE.equals(e.getType())).findAny();
|
||||
manualType.ifPresent(typeEntity -> typeRepository.deleteById(typeEntity.getId()));
|
||||
|
||||
var rank = 1;
|
||||
for (var entity : entitiesForDossierTemplate) {
|
||||
if (entity.getRank() > rank) {
|
||||
rank = entity.getRank();
|
||||
}
|
||||
}
|
||||
rank += 1000;
|
||||
|
||||
dictionaryPersistenceService.addType(MANUAL_TYPE,
|
||||
dossierTemplateEntity.getId(),
|
||||
"#9398a0",
|
||||
"#c5d3eb",
|
||||
"#c498fa",
|
||||
rank,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
"Manual Redactions",
|
||||
false,
|
||||
"Manual Redactions",
|
||||
null,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
true);
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,86 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class MigrateHighlights3 extends Migration {
|
||||
|
||||
private static final String NAME = "Migrate Highlights";
|
||||
private static final long VERSION = 3;
|
||||
|
||||
@Autowired
|
||||
private DossierPersistenceService dossierPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private FileStatusPersistenceService fileStatusPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
@Autowired
|
||||
private FileStatusService fileStatusService;
|
||||
|
||||
|
||||
public MigrateHighlights3() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
var dossiers = dossierPersistenceService.findAllDossiers();
|
||||
dossiers.forEach(dossier -> {
|
||||
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId());
|
||||
files.forEach(file -> {
|
||||
if (file.getHardDeletedTime() == null) {
|
||||
|
||||
try {
|
||||
if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) {
|
||||
var untouchedExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.UNTOUCHED);
|
||||
|
||||
if (!untouchedExists) {
|
||||
var originExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN);
|
||||
|
||||
if (!originExists) {
|
||||
log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
fileManagementStorageService.storeObject(dossier.getId(),
|
||||
file.getId(),
|
||||
FileType.UNTOUCHED,
|
||||
new ByteArrayInputStream(fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN)));
|
||||
|
||||
}
|
||||
|
||||
fileStatusService.addToPreprocessingQueue(file.getId(), file.getDossierId(), file.getFilename());
|
||||
|
||||
} else {
|
||||
log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to extract text highlights for document: {}", file.getId());
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,78 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class RemoveFalsePositiveManualRedactions6 extends Migration {
|
||||
|
||||
private static final String NAME = "Remove false positives manual redactions";
|
||||
private static final long VERSION = 6;
|
||||
|
||||
@Autowired
|
||||
private DossierPersistenceService dossierPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private FileStatusPersistenceService fileStatusPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private ManualRedactionService manualRedactionService;
|
||||
|
||||
@Autowired
|
||||
private AddRedactionPersistenceService addRedactionPersistenceService;
|
||||
|
||||
|
||||
public RemoveFalsePositiveManualRedactions6() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
var dossiers = dossierPersistenceService.findAllDossiers();
|
||||
dossiers.forEach(dossier -> {
|
||||
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId());
|
||||
files.forEach(file -> {
|
||||
if (file.getHardDeletedTime() == null) {
|
||||
var annotationIdsToRemove = new ArrayList<String>();
|
||||
var manualRedactions = manualRedactionService.getManualRedactions( file.getId());
|
||||
|
||||
if (manualRedactions != null && manualRedactions.getEntriesToAdd() != null) {
|
||||
manualRedactions.getEntriesToAdd().forEach(addRedaction -> {
|
||||
if (addRedaction.getType().contains("false_positive")) {
|
||||
annotationIdsToRemove.add(addRedaction.getAnnotationId());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(), annotationIdsToRemove);
|
||||
|
||||
if (!annotationIdsToRemove.isEmpty()) {
|
||||
annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id));
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,84 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.model.simplifiedtext.Text;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class SimplifiedTextMigration12 extends Migration {
|
||||
|
||||
private static final String NAME = "Add simplified text file if missing";
|
||||
private static final long VERSION = 12;
|
||||
|
||||
@Autowired
|
||||
private DossierPersistenceService dossierPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private FileStatusPersistenceService fileStatusPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private StorageService storageService;
|
||||
|
||||
|
||||
public SimplifiedTextMigration12() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
var dossiers = dossierPersistenceService.findAllDossiers();
|
||||
dossiers.forEach(dossier -> {
|
||||
if (dossier.getHardDeletedTime() == null) {
|
||||
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId());
|
||||
log.info("Start migration of dossier {}", dossier.getId());
|
||||
files.forEach(file -> {
|
||||
if (file.getHardDeletedTime() == null) {
|
||||
log.info("Start migration of file {}", file.getId());
|
||||
migrateFile(dossier.getId(), file.getId());
|
||||
log.info("Finished migration of file {}", file.getId());
|
||||
}
|
||||
});
|
||||
log.info("Finished migration of dossier {}", dossier.getId());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void migrateFile(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
|
||||
if (!storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT))) {
|
||||
var text = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
|
||||
storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT), text);
|
||||
}
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.warn("Text not found for dossier {} and file {}, ignoring....", dossierId, fileId);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,142 +0,0 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
||||
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class TypeToEntityMigration5 extends Migration {
|
||||
|
||||
private static final String NAME = "Migrate false positives, set recommendation color, remove unneeded dictionaries";
|
||||
private static final long VERSION = 5;
|
||||
|
||||
@Autowired
|
||||
private DictionaryPersistenceService dictionaryPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private DossierTemplateRepository dossierTemplateRepository;
|
||||
|
||||
@Autowired
|
||||
private EntryPersistenceService entryPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private DictionaryManagementService dictionaryManagementService;
|
||||
|
||||
|
||||
public TypeToEntityMigration5() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
var types = dictionaryPersistenceService.getAllTypes(false);
|
||||
types.forEach(type -> type.setRecommendationHexColor("#8df06c"));
|
||||
dictionaryPersistenceService.saveAllTypes(types);
|
||||
|
||||
var dossierTemplates = dossierTemplateRepository.findAllWhereDeletedIsFalse();
|
||||
|
||||
dossierTemplates.forEach(dossierTemplate -> {
|
||||
log.info("Starting false positive migration of dossierTemplate {}", dossierTemplate.getId());
|
||||
var typeIdsToDelete = new HashSet<String>();
|
||||
var typesOfDossierTemplate = dictionaryPersistenceService.getAllTypes(false);
|
||||
var falsePositive = typesOfDossierTemplate.stream().filter(t -> t.getType().equals("false_positive")).findFirst();
|
||||
if (falsePositive.isEmpty()) {
|
||||
log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName());
|
||||
return;
|
||||
}
|
||||
typeIdsToDelete.add(falsePositive.get().getId());
|
||||
var falsePositiveEntries = entryPersistenceService.getEntries(falsePositive.get().getId(), DictionaryEntryType.ENTRY, null)
|
||||
.stream()
|
||||
.map(BaseDictionaryEntry::getValue)
|
||||
.collect(Collectors.toSet());
|
||||
log.info("False positive migration of dossierTemplate {} has {} false positive entries", dossierTemplate.getId(), falsePositiveEntries.size());
|
||||
|
||||
typesOfDossierTemplate.stream().filter(t -> !t.getType().equals("false_positive")).forEach(typeEntity -> {
|
||||
log.info("Start processing type {}", typeEntity.getType());
|
||||
|
||||
if (typeEntity.getType().startsWith("recommendation_")) {
|
||||
typeIdsToDelete.add(typeEntity.getId());
|
||||
log.info("Finished processing type {}", typeEntity.getType());
|
||||
return;
|
||||
}
|
||||
|
||||
var entries = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.ENTRY, null)
|
||||
.stream()
|
||||
.map(BaseDictionaryEntry::getValue)
|
||||
.collect(Collectors.toSet());
|
||||
log.info("type {} has {} entries", typeEntity.getType(), entries);
|
||||
|
||||
var typeFalsePositives = new HashSet<String>();
|
||||
falsePositiveEntries.forEach(falsePositiveValue -> {
|
||||
if (containsAnyWord(falsePositiveValue, entries, typeEntity.isCaseInsensitive())) {
|
||||
typeFalsePositives.add(falsePositiveValue);
|
||||
}
|
||||
});
|
||||
|
||||
if (!typeFalsePositives.isEmpty()) {
|
||||
dictionaryManagementService.addEntries(typeEntity.getId(), new ArrayList<>(typeFalsePositives), false, false, DictionaryEntryType.FALSE_POSITIVE);
|
||||
log.info("Added {} for type {}", typeFalsePositives.size(), typeEntity.getType());
|
||||
}
|
||||
log.info("Finished processing type {}", typeEntity.getType());
|
||||
});
|
||||
|
||||
typeIdsToDelete.forEach(typeIdToDelete -> dictionaryManagementService.deleteType(typeIdToDelete));
|
||||
log.info("Finished false positive migration of dossierTemplate {}", dossierTemplate.getId());
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
public boolean containsAnyWord(String text, Set<String> values, boolean isCaseInsensitive) {
|
||||
|
||||
String inputString = isCaseInsensitive ? text.toLowerCase(Locale.ROOT) : text;
|
||||
|
||||
for (String value : values) {
|
||||
|
||||
String cleanValue = isCaseInsensitive ? value.toLowerCase(Locale.ROOT).trim() : value.trim();
|
||||
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = inputString.indexOf(cleanValue, stopIndex);
|
||||
stopIndex = startIndex + cleanValue.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(
|
||||
inputString.charAt(stopIndex)))) {
|
||||
return true;
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user