Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #510: RED-4861: Do no full analysis for resize/remove/force

Browse Source 
Merge in RED/persistence-service from RED-4861 to master

* commit '5ed1718441d57f1a91cbe78590369ac6b0f8cd9b':
  RED-4861: Do no full analysis for resize/remove/force
This commit is contained in:
Dominique Eiflaender 2022-08-04 13:16:49 +02:00
commit 058e22d4be
1 changed files with 53 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -82,7 +82,10 @@ public class FileStatusService {
var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(fileManagementServiceSettings.getMaxErrorRetries());
var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList)
.stream()
.filter(FileModel::isAnalysisRequired)
.collect(Collectors.toList());
}
@ -121,7 +124,8 @@ public class FileStatusService {
}
public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) {
public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights,
long fileSize) {
fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize);
addToAnalysisQueue(dossierId, fileId, false, Set.of());
@ -176,7 +180,8 @@ public class FileStatusService {
@Transactional
public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually) {
public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse,
boolean triggeredManually) {
log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
@ -210,7 +215,8 @@ public class FileStatusService {
@Transactional
public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) {
public void setStatusFullReprocess(String dossierId, String fileId, boolean priority,
boolean requiresStructureAnalysis) {
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
@ -232,7 +238,11 @@ public class FileStatusService {
@SneakyThrows
public void addToPreprocessingQueue(String dossierId, String fileId, String filename) {
var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder().dossierId(dossierId).fileId(fileId).fileName(filename).build();
var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.fileName(filename)
.build();
setStatusPreProcessing(fileId);
@ -421,7 +431,8 @@ public class FileStatusService {
@Transactional
public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean keepManualRedactions) {
public void overwriteFile(String dossierId, String fileId, String uploader, String filename,
boolean keepManualRedactions) {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ORIGIN);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
@ -458,7 +469,8 @@ public class FileStatusService {
// wipe force redactions
var forceRedactions = forceRedactionPersistenceService.findForceRedactions(fileId, false);
forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId()
.getAnnotationId(), now));
// wipe add manual redactions
var addRedactions = addRedactionPersistenceService.findAddRedactions(fileId, false);
@ -466,19 +478,23 @@ public class FileStatusService {
// wipe removeRedactions
var removeRedactions = removeRedactionPersistenceService.findRemoveRedactions(fileId, false);
removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId()
.getAnnotationId(), now));
// wipe image recat
var imageRecategorizations = imageRecategorizationPersistenceService.findRecategorizations(fileId, false);
imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));// wipe image recat
imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId()
.getAnnotationId(), now));// wipe image recat
// wipe resize redactions
var resizeRedactions = resizeRedactionPersistenceService.findResizeRedactions(fileId, false);
resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId()
.getAnnotationId(), now));
// wipe legal basis changes
var legalBasisChanges = legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false);
legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId()
.getAnnotationId(), now));
fileStatusPersistenceService.updateHasComments(fileId, false);
}
@ -512,7 +528,8 @@ public class FileStatusService {
@Transactional
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse) {
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority,
Set<Integer> sectionsToReanalyse) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
@ -598,7 +615,8 @@ public class FileStatusService {
}
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) {
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
ProcessingStatus processingStatus, FileModel fileModel) {
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
return MessageType.STRUCTURE_ANALYSE;
@ -607,20 +625,35 @@ public class FileStatusService {
return MessageType.ANALYSE;
}
if (reanalyse) {
if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) {
if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed()
.isBefore(fileModel.getLastFileAttributeChange())) {
return MessageType.ANALYSE;
}
return MessageType.REANALYSE;
} else if (fileModel.getLastProcessed() != null && manualRedactionProviderService.getManualRedactions(fileId)
.getResizeRedactions()
.stream()
.anyMatch(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed()))) {
} else if (hasUnprocessedManualRedactions(fileModel)) {
return MessageType.REANALYSE;
}
return MessageType.ANALYSE;
}
private boolean hasUnprocessedManualRedactions(FileModel fileModel) {
return fileModel.getLastProcessed() != null && (manualRedactionProviderService.getManualRedactions(fileModel.getId())
.getResizeRedactions()
.stream()
.anyMatch(manual -> manual.getRequestDate()
.isAfter(fileModel.getLastProcessed())) || manualRedactionProviderService.getManualRedactions(fileModel.getId())
.getForceRedactions()
.stream()
.anyMatch(manual -> manual.getRequestDate()
.isAfter(fileModel.getLastProcessed())) || manualRedactionProviderService.getManualRedactions(fileModel.getId())
.getIdsToRemove()
.stream()
.anyMatch(manual -> manual.getRequestDate().isAfter(fileModel.getLastProcessed())));
}
private void addToOcrQueue(String dossierId, String fileId, int priority) {
try {
@ -634,7 +667,8 @@ public class FileStatusService {
}
private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(
List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
List<com.iqser.red.service.redaction.v1.model.FileAttribute> fileAttributeList = new ArrayList<>();