Pull request #457: RED-4343: Added processingErrorCounter to FileEntity and limit retries for failure files to 5 (via setting)
Merge in RED/persistence-service from RED-4343 to master * commit '4d324347e39d9ac413506ef21f1706383633d6b6': RED-4343: Added processingErrorCounter to FileEntity and limit retries for failure files to 5 (via setting)
This commit is contained in:
Philipp Schramm
commit
71a34f58a2
@ -33,8 +33,7 @@ public interface FileStatusProcessingUpdateResource {
|
||||
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
@PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-successful", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId,
|
||||
@RequestBody AnalyzeResult analyzeResult);
|
||||
void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AnalyzeResult analyzeResult);
|
||||
|
||||
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
|
||||
@ -1,13 +1,18 @@
|
||||
package com.iqser.red.service.persistence.service.v1.api.resources;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.DeleteImportedRedactionsRequest;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
@ResponseStatus(value = HttpStatus.NO_CONTENT)
|
||||
public interface ReanalysisResource {
|
||||
@ -55,9 +60,11 @@ public interface ReanalysisResource {
|
||||
@PostMapping(value = IMPORT_REDACTIONS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void importRedactions(@RequestBody ByteContentDocument documentRequest);
|
||||
|
||||
|
||||
@PostMapping(value = IMPORT_REDACTIONS_PATH + "/delete", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void deleteImportedRedactions(@RequestBody DeleteImportedRedactionsRequest deleteImportedRedactionsRequest);
|
||||
|
||||
|
||||
@PostMapping(value = CONVERT_TEXT_HIGHLIGHTS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest);
|
||||
|
||||
|
||||
@ -169,6 +169,9 @@ public class FileEntity {
|
||||
@Column(name = "dossier_id")
|
||||
private String dossierId;
|
||||
|
||||
@Column
|
||||
private Integer processingErrorCounter;
|
||||
|
||||
public boolean isSoftDeleted() {
|
||||
return deleted != null;
|
||||
}
|
||||
|
||||
@ -52,6 +52,7 @@ public class FileStatusPersistenceService {
|
||||
file.setLastUploaded(now);
|
||||
file.setLastUpdated(now);
|
||||
file.setFileManipulationDate(now);
|
||||
file.setProcessingErrorCounter(0);
|
||||
fileRepository.save(file);
|
||||
}
|
||||
|
||||
@ -62,8 +63,8 @@ public class FileStatusPersistenceService {
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED,
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights, fileSize);
|
||||
fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), hasHighlights, fileSize, calculateProcessingErrorCounter(fileId, ProcessingStatus.PRE_PROCESSED));
|
||||
}
|
||||
|
||||
|
||||
@ -73,27 +74,26 @@ public class FileStatusPersistenceService {
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_FAILED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_FAILED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PRE_PROCESSING_FAILED));
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion,
|
||||
long legalBasisVersion, long duration, long dossierDictionaryVersion,
|
||||
int analysisVersion, int analysisNumber) {
|
||||
public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion, long legalBasisVersion, long duration,
|
||||
long dossierDictionaryVersion, int analysisVersion, int analysisNumber) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion,
|
||||
legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
|
||||
fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), analysisNumber, calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages,
|
||||
boolean hasSuggestions, boolean hasComments, boolean hasUpdates) {
|
||||
public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
@ -102,12 +102,6 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
private boolean isFileDeleted(String fileId) {
|
||||
|
||||
return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, boolean approval) {
|
||||
|
||||
@ -115,12 +109,10 @@ public class FileStatusPersistenceService {
|
||||
return;
|
||||
}
|
||||
if (approval) {
|
||||
fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
|
||||
fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), true);
|
||||
} else {
|
||||
fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), null);
|
||||
fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), null);
|
||||
}
|
||||
|
||||
}
|
||||
@ -134,11 +126,11 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
if (processingStatus == ProcessingStatus.PROCESSED) {
|
||||
// In case the file is updated to "processed", "lastProcessed" date should be updated to "now"
|
||||
fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
|
||||
} else {
|
||||
fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
.truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, processingStatus));
|
||||
}
|
||||
}
|
||||
|
||||
@ -170,8 +162,8 @@ public class FileStatusPersistenceService {
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
|
||||
}
|
||||
|
||||
|
||||
@ -236,12 +228,10 @@ public class FileStatusPersistenceService {
|
||||
|
||||
public List<FileEntity> getActiveFiles(String dossierId) {
|
||||
|
||||
return fileRepository.findByDossierId(dossierId)
|
||||
.stream()
|
||||
.filter(f -> !f.isSoftOrHardDeleted())
|
||||
.collect(Collectors.toList());
|
||||
return fileRepository.findByDossierId(dossierId).stream().filter(f -> !f.isSoftOrHardDeleted()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
public List<FileEntity> getSoftDeletedFiles(List<String> dossierIds) {
|
||||
|
||||
return fileRepository.getSoftDeletedFiles(dossierIds);
|
||||
@ -264,8 +254,7 @@ public class FileStatusPersistenceService {
|
||||
@Transactional
|
||||
public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
|
||||
|
||||
int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
|
||||
int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
|
||||
if (countUpdate == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
@ -282,8 +271,7 @@ public class FileStatusPersistenceService {
|
||||
}, () -> {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
});
|
||||
fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), null);
|
||||
fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), null);
|
||||
}
|
||||
|
||||
|
||||
@ -293,8 +281,7 @@ public class FileStatusPersistenceService {
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
if (updateCount == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
@ -307,8 +294,7 @@ public class FileStatusPersistenceService {
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
if (countUpdate == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
@ -321,8 +307,7 @@ public class FileStatusPersistenceService {
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
if (countUpdate == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
@ -332,18 +317,17 @@ public class FileStatusPersistenceService {
|
||||
@Transactional
|
||||
public void overwriteFile(String fileId, String uploader, String filename, boolean keepManualRedactions) {
|
||||
|
||||
int countUpdate = 0;
|
||||
int countUpdate;
|
||||
|
||||
if(keepManualRedactions) {
|
||||
if (keepManualRedactions) {
|
||||
countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
} else {
|
||||
|
||||
fileRepository.findById(fileId).ifPresent((file) -> {
|
||||
file.setExcludedPages(new HashSet<>());
|
||||
});
|
||||
fileRepository.findById(fileId).ifPresent((file) -> file.setExcludedPages(new HashSet<>()));
|
||||
|
||||
countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, WorkflowStatus.NEW, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
}
|
||||
|
||||
deleteFileAttributes(fileId);
|
||||
@ -354,33 +338,13 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
private List<FileAttributeEntity> convertFileAttributes(String dossierId, FileEntity file,
|
||||
Map<String, String> fileAttributesMap) {
|
||||
|
||||
var dossier = dossierService.getAndValidateDossier(dossierId);
|
||||
List<FileAttributeConfigEntity> configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId());
|
||||
|
||||
return fileAttributesMap.entrySet().stream().map(entry -> {
|
||||
var fa = new FileAttributeEntity();
|
||||
var id = new FileAttributeEntity.FileAttributeEntityId();
|
||||
id.setFileId(file.getId());
|
||||
id.setFileAttributeConfigId(configuration.stream()
|
||||
.filter(c -> c.getId().equals(entry.getKey()))
|
||||
.findAny().map(FileAttributeConfigEntity::getId)
|
||||
.orElseThrow(() -> new BadRequestException("Invalid File Attribute Id")));
|
||||
fa.setFileAttributeId(id);
|
||||
fa.setValue(StringUtils.isBlank(entry.getValue()) ? null : entry.getValue());
|
||||
return fa;
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime changeDate) {
|
||||
|
||||
fileRepository.setLastRedactionModificationDateForFile(fileId, changeDate, OffsetDateTime.now());
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate) {
|
||||
|
||||
@ -399,6 +363,7 @@ public class FileStatusPersistenceService {
|
||||
return fileRepository.countSoftDeletedFilesPerDossierTemplateId(dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
public int countSoftDeletedFilesPerDossierId(String dossierId) {
|
||||
|
||||
return fileRepository.countSoftDeletedFilesPerDossierId(dossierId);
|
||||
@ -424,4 +389,48 @@ public class FileStatusPersistenceService {
|
||||
fileRepository.updateHasHighlights(fileId, hasHighlights);
|
||||
}
|
||||
|
||||
|
||||
private int calculateProcessingErrorCounter(String fileId, ProcessingStatus processingStatus) {
|
||||
|
||||
switch (processingStatus) {
|
||||
case ERROR:
|
||||
case PRE_PROCESSING_FAILED:
|
||||
return fileRepository.findById(fileId).map(FileEntity::getProcessingErrorCounter).orElse(0) + 1;
|
||||
|
||||
case PROCESSED:
|
||||
return 0;
|
||||
|
||||
default:
|
||||
return fileRepository.findById(fileId).map(FileEntity::getProcessingErrorCounter).orElse(0);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private boolean isFileDeleted(String fileId) {
|
||||
|
||||
return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
|
||||
}
|
||||
|
||||
|
||||
private List<FileAttributeEntity> convertFileAttributes(String dossierId, FileEntity file, Map<String, String> fileAttributesMap) {
|
||||
|
||||
var dossier = dossierService.getAndValidateDossier(dossierId);
|
||||
List<FileAttributeConfigEntity> configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId());
|
||||
|
||||
return fileAttributesMap.entrySet().stream().map(entry -> {
|
||||
var fa = new FileAttributeEntity();
|
||||
var id = new FileAttributeEntity.FileAttributeEntityId();
|
||||
id.setFileId(file.getId());
|
||||
id.setFileAttributeConfigId(configuration.stream()
|
||||
.filter(c -> c.getId().equals(entry.getKey()))
|
||||
.findAny()
|
||||
.map(FileAttributeConfigEntity::getId)
|
||||
.orElseThrow(() -> new BadRequestException("Invalid File Attribute Id")));
|
||||
fa.setFileAttributeId(id);
|
||||
fa.setValue(StringUtils.isBlank(entry.getValue()) ? null : entry.getValue());
|
||||
return fa;
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,88 +1,73 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
public interface FileRepository extends JpaRepository<FileEntity, String> {
|
||||
|
||||
boolean existsByDossierIdAndLastUpdatedIsAfter(String dossierId, OffsetDateTime since);
|
||||
|
||||
|
||||
List<FileEntity> findByDossierId(String dossierId);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity e set e.hasRedactions = :hasRedactions ," +
|
||||
" e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," +
|
||||
" e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " +
|
||||
" e.lastUpdated = :lastUpdated " +
|
||||
" where e.id =:fileId")
|
||||
void updateFlags(String fileId,
|
||||
OffsetDateTime lastUpdated,
|
||||
boolean hasRedactions,
|
||||
boolean hasHints,
|
||||
boolean hasImages,
|
||||
boolean hasSuggestions,
|
||||
boolean hasComments,
|
||||
@Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + " e.lastUpdated = :lastUpdated " + " where e.id =:fileId")
|
||||
void updateFlags(String fileId, OffsetDateTime lastUpdated, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments,
|
||||
boolean hasUpdates);
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " +
|
||||
"f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " +
|
||||
"f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " +
|
||||
"f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " +
|
||||
"f.lastProcessed = :lastProcessed " +
|
||||
"where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion,
|
||||
long rulesVersion, long legalBasisVersion, long analysisDuration, long dossierDictionaryVersion,
|
||||
int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber);
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," +
|
||||
" f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId")
|
||||
@Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion, long rulesVersion, long legalBasisVersion,
|
||||
long analysisDuration, long dossierDictionaryVersion, int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed,
|
||||
int analysisNumber, int processingErrorCounter);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId")
|
||||
void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate, boolean excludedFromAutomaticAnalysis);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " +
|
||||
" where f.id = :fileId")
|
||||
@Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " + " where f.id = :fileId")
|
||||
void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
|
||||
" f.hasHighlights = :hasHighlights, f.fileSize = :fileSize " +
|
||||
" where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.fileSize = :fileSize where f.id = :fileId")
|
||||
void updateFileSize(String fileId, long fileSize);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated " +
|
||||
"where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated);
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + " f.hasHighlights = :hasHighlights, f.fileSize = :fileSize, f.processingErrorCounter = :processingErrorCounter " + " where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize, int processingErrorCounter);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed " +
|
||||
"where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed);
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
|
||||
"f.lastIndexed = :lastIndexed where f.id = :fileId")
|
||||
void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed);
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int processingErrorCounter);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId")
|
||||
void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed, int processingErrorCounter);
|
||||
|
||||
|
||||
@Modifying
|
||||
@ -107,117 +92,91 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id = :fileId")
|
||||
int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
|
||||
OffsetDateTime softDeletedTime);
|
||||
int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime softDeletedTime);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
|
||||
"f.hardDeletedTime = :hardDeletedTime, " +
|
||||
"f.deleted = case " +
|
||||
"when f.deleted is null then :deleted " +
|
||||
"when f.deleted is not null then f.deleted " +
|
||||
"end " +
|
||||
"where f.id = :fileId")
|
||||
int setHardDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
|
||||
OffsetDateTime hardDeletedTime, OffsetDateTime deleted);
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.hardDeletedTime = :hardDeletedTime, " + "f.deleted = case " + "when f.deleted is null then :deleted " + "when f.deleted is not null then f.deleted " + "end " + "where f.id = :fileId")
|
||||
int setHardDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime hardDeletedTime, OffsetDateTime deleted);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update FileEntity f set f.assignee = :assignee, f.lastReviewer = :lastReviewer, f.lastApprover = :lastApprover, " +
|
||||
"f.lastUpdated = :lastUpdated where f.id = :fileId")
|
||||
@Query("update FileEntity f set f.assignee = :assignee, f.lastReviewer = :lastReviewer, f.lastApprover = :lastApprover, " + "f.lastUpdated = :lastUpdated where f.id = :fileId")
|
||||
int setAssignee(String fileId, String assignee, String lastReviewer, String lastApprover, OffsetDateTime lastUpdated);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.excluded = :excluded, f.lastUpdated = :lastUpdated where f.id = :fileId")
|
||||
int toggleExclusion(String fileId, boolean excluded, OffsetDateTime lastUpdated);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis, f.lastUpdated = :lastUpdated where f.id = :fileId")
|
||||
int toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis, OffsetDateTime lastUpdated);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, "
|
||||
+ "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, "
|
||||
+ "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, "
|
||||
+ "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
|
||||
+ "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
|
||||
+ "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
|
||||
+ "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false " + "where f.id = :fileId")
|
||||
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, f.processingErrorCounter = 0 where f.id = :fileId")
|
||||
int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded,
|
||||
OffsetDateTime lastUpdated);
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, "
|
||||
+ "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, "
|
||||
+ "f.lastOCRTime = null, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, "
|
||||
+ "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
|
||||
+ "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
|
||||
+ "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
|
||||
+ "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false " + "where f.id = :fileId")
|
||||
int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded,
|
||||
OffsetDateTime lastUpdated);
|
||||
|
||||
@Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" +
|
||||
" and ((f.deleted is not null and f.hardDeletedTime is null) or " +
|
||||
" (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0 where f.id = :fileId")
|
||||
int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded,
|
||||
OffsetDateTime lastUpdated);
|
||||
|
||||
|
||||
@Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
|
||||
int countSoftDeletedFilesPerDossierTemplateId(String dossierTemplateId);
|
||||
|
||||
@Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.id = :dossierId" +
|
||||
" and ((f.deleted is not null and f.hardDeletedTime is null) or " +
|
||||
" (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
|
||||
|
||||
@Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.id = :dossierId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
|
||||
int countSoftDeletedFilesPerDossierId(String dossierId);
|
||||
|
||||
|
||||
@Query("select distinct f.dossierId from FileEntity f where f.lastUpdated > :since")
|
||||
List<String> findDossierChangeByLastUpdatedIsAfter(OffsetDateTime since);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.redactionModificationDate = :redactionModificationDate, f.lastUpdated = :lastUpdated where f.id = :fileId")
|
||||
void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime redactionModificationDate, OffsetDateTime lastUpdated);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.lastManualChangeDate = :lastManualChangeDate, f.lastUpdated = :lastUpdated where f.id = :fileId")
|
||||
void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate, OffsetDateTime lastUpdated);
|
||||
|
||||
@Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " +
|
||||
"and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'ERROR' )" +
|
||||
"and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " +
|
||||
"and f.deleted is null and f.hardDeletedTime is null")
|
||||
|
||||
@Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + "and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'ERROR' )" + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and f.deleted is null and f.hardDeletedTime is null")
|
||||
List<FileEntity> getAllRelevantStatusesForReanalysisScheduler();
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.lastFileAttributeChange = :date, f.lastUpdated = :date where f.id = :fileId")
|
||||
void updateLastAttributeChangeDate(String fileId, OffsetDateTime date);
|
||||
|
||||
|
||||
@Query("select f from FileEntity f where f.deleted is not null and f.hardDeletedTime is null and f.dossierId in :dossierIds")
|
||||
List<FileEntity> getSoftDeletedFiles(List<String> dossierIds);
|
||||
|
||||
@Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " +
|
||||
"inner join DossierEntity d on d.id = f.dossierId " +
|
||||
"where f.deleted is null and f.hardDeletedTime is null " +
|
||||
"and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " +
|
||||
"and d.dossierTemplateId = :dossierTemplateId " +
|
||||
"group by f.processingStatus ")
|
||||
|
||||
@Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.processingStatus ")
|
||||
List<FileProcessingStatusProjection> countFilesByProcessingStatus(String dossierTemplateId);
|
||||
|
||||
@Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " +
|
||||
"inner join DossierEntity d on d.id = f.dossierId " +
|
||||
"where f.deleted is null and f.hardDeletedTime is null " +
|
||||
"and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " +
|
||||
"and d.dossierTemplateId = :dossierTemplateId " +
|
||||
"group by f.workflowStatus ")
|
||||
|
||||
@Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.workflowStatus ")
|
||||
List<FileWorkflowStatusProjection> countFilesByWorkflowStatus(String dossierTemplateId);
|
||||
|
||||
@Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " +
|
||||
" from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " +
|
||||
" and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" +
|
||||
" and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true)
|
||||
|
||||
@Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " + " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true)
|
||||
FilePageCountsProjection countPages(String dossierTemplateId);
|
||||
|
||||
@Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " +
|
||||
"f.hardDeletedTime is null and f.deleted is null and " +
|
||||
"d.dossierTemplateId = :dossierTemplateId and " +
|
||||
"d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null")
|
||||
|
||||
@Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + "f.hardDeletedTime is null and f.deleted is null and " + "d.dossierTemplateId = :dossierTemplateId and " + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null")
|
||||
int countActiveFiles(String dossierTemplateId);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,8 +1,21 @@
|
||||
package com.iqser.red.service.peristence.v1.server.controller;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.*;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
@ -15,24 +28,12 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.DeleteImportedRedactionsRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.resources.ReanalysisResource;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@ -48,8 +49,7 @@ public class ReanalysisController implements ReanalysisResource {
|
||||
|
||||
|
||||
@Override
|
||||
public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
|
||||
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
|
||||
public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
|
||||
|
||||
var relevantFiles = getAllFilesForDossier(dossierId, true);
|
||||
reanalyseFiles(dossierId, force, relevantFiles);
|
||||
@ -94,8 +94,7 @@ public class ReanalysisController implements ReanalysisResource {
|
||||
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
|
||||
} else {
|
||||
FileModel dossierFile = fileStatusService.getStatus(fileId);
|
||||
if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus()
|
||||
.equals(WorkflowStatus.APPROVED)) {
|
||||
if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
|
||||
throw new ConflictException("Cannot analyse a deleted/approved file");
|
||||
}
|
||||
if (dossierFile.getLastOCRTime() != null) {
|
||||
@ -107,6 +106,7 @@ public class ReanalysisController implements ReanalysisResource {
|
||||
|
||||
|
||||
public void importRedactions(@RequestBody ByteContentDocument documentRequest) {
|
||||
|
||||
try {
|
||||
pDFTronRedactionClient.importRedactions(documentRequest);
|
||||
} catch (Exception e) {
|
||||
@ -129,8 +129,7 @@ public class ReanalysisController implements ReanalysisResource {
|
||||
|
||||
|
||||
public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId,
|
||||
@RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex,
|
||||
@RequestBody Set<String> fileIds) {
|
||||
@RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, @RequestBody Set<String> fileIds) {
|
||||
|
||||
indexingService.reindex(dossierId, fileIds, dropIndex);
|
||||
}
|
||||
@ -141,8 +140,7 @@ public class ReanalysisController implements ReanalysisResource {
|
||||
boolean hasTextHighlights = pDFTronRedactionClient.convertTextHighlights(textHighlightRequest);
|
||||
|
||||
fileStatusService.updateHasHighlights(textHighlightRequest.getFileId(), hasTextHighlights);
|
||||
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
|
||||
if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) {
|
||||
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false);
|
||||
|
||||
@ -1,17 +1,17 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service;
|
||||
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
|
||||
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.search.v1.model.IndexMessageType;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@ -43,7 +43,7 @@ public class FileStatusProcessingUpdateService {
|
||||
fileStatusService.setStatusAnalyse(dossierId, fileId, false);
|
||||
|
||||
//TODO This might be also priority depending on what was the pervious call.
|
||||
fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null);
|
||||
fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null, false);
|
||||
}
|
||||
break;
|
||||
|
||||
@ -72,20 +72,18 @@ public class FileStatusProcessingUpdateService {
|
||||
}
|
||||
|
||||
|
||||
public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse){
|
||||
public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse) {
|
||||
|
||||
fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(),untouchedDocumentResponse.getFileSize());
|
||||
fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(), untouchedDocumentResponse.getFileSize());
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void preprocessingFailed(String dossierId, String fileId){
|
||||
public void preprocessingFailed(String dossierId, String fileId) {
|
||||
|
||||
fileStatusService.updateProcessingStatusPreprocessingFailed(dossierId, fileId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void ocrSuccessful(String dossierId, String fileId) {
|
||||
|
||||
retryTemplate.execute(retryContext -> {
|
||||
@ -111,16 +109,6 @@ public class FileStatusProcessingUpdateService {
|
||||
}
|
||||
|
||||
|
||||
private void setStatusError(String dossierId, String fileId, String reason) {
|
||||
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason);
|
||||
fileStatusService.setStatusError(fileId);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public void indexing(String dossierId, String fileId) {
|
||||
|
||||
retryTemplate.execute(retryContext -> {
|
||||
@ -146,4 +134,14 @@ public class FileStatusProcessingUpdateService {
|
||||
setStatusError(dossierId, fileId, "indexingFailed");
|
||||
}
|
||||
|
||||
|
||||
private void setStatusError(String dossierId, String fileId, String reason) {
|
||||
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason);
|
||||
fileStatusService.setStatusError(fileId);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,5 +1,19 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Sets;
|
||||
@ -11,7 +25,6 @@ import com.iqser.red.service.peristence.v1.server.model.Pdf2ImageServiceRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
|
||||
@ -20,7 +33,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
@ -28,21 +47,10 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.MessageType;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@ -75,10 +83,7 @@ public class FileStatusService {
|
||||
|
||||
var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler();
|
||||
var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
|
||||
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList)
|
||||
.stream()
|
||||
.filter(FileModel::isAnalysisRequired)
|
||||
.collect(Collectors.toList());
|
||||
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@ -119,8 +124,8 @@ public class FileStatusService {
|
||||
|
||||
public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) {
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights,fileSize);
|
||||
addToAnalysisQueue(dossierId, fileId, false, Set.of());
|
||||
fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize);
|
||||
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
|
||||
|
||||
if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) {
|
||||
addToPdf2ImageQueue(dossierId, fileId);
|
||||
@ -182,8 +187,7 @@ public class FileStatusService {
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse,
|
||||
boolean triggeredManually) {
|
||||
public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually) {
|
||||
|
||||
log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
|
||||
|
||||
@ -199,8 +203,13 @@ public class FileStatusService {
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries() && !triggeredManually) {
|
||||
log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter());
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS);
|
||||
addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse);
|
||||
addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse, triggeredManually);
|
||||
}
|
||||
|
||||
|
||||
@ -217,8 +226,7 @@ public class FileStatusService {
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusFullReprocess(String dossierId, String fileId, boolean priority,
|
||||
boolean requiresStructureAnalysis) {
|
||||
public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) {
|
||||
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
@ -227,24 +235,25 @@ public class FileStatusService {
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) {
|
||||
log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter());
|
||||
return;
|
||||
}
|
||||
|
||||
if (requiresStructureAnalysis) {
|
||||
log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
|
||||
}
|
||||
|
||||
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
|
||||
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void addToPreprocessingQueue(String dossierId, String fileId, String filename) {
|
||||
|
||||
var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.fileName(filename)
|
||||
.build();
|
||||
var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder().dossierId(dossierId).fileId(fileId).fileName(filename).build();
|
||||
|
||||
setStatusPreProcessing(fileId);
|
||||
|
||||
@ -253,8 +262,7 @@ public class FileStatusService {
|
||||
|
||||
|
||||
@Transactional
|
||||
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority,
|
||||
Set<Integer> sectionsToReanalyse) {
|
||||
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually) {
|
||||
|
||||
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
|
||||
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
|
||||
@ -264,6 +272,11 @@ public class FileStatusService {
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileEntity.getProcessingErrorCounter() >= settings.getMaxErrorRetries() && !triggeredManually) {
|
||||
log.warn("File {} was {} times retried with failure", fileEntity.getId(), fileEntity.getProcessingErrorCounter());
|
||||
return;
|
||||
}
|
||||
|
||||
var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
|
||||
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
|
||||
|
||||
@ -316,28 +329,6 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
|
||||
ProcessingStatus processingStatus, FileModel fileModel) {
|
||||
|
||||
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
|
||||
return MessageType.STRUCTURE_ANALYSE;
|
||||
}
|
||||
if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
|
||||
return MessageType.ANALYSE;
|
||||
}
|
||||
if (reanalyse) {
|
||||
if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed()
|
||||
.isBefore(fileModel.getLastFileAttributeChange())) {
|
||||
return MessageType.ANALYSE;
|
||||
}
|
||||
return MessageType.REANALYSE;
|
||||
} else if(manualRedactionProviderService.getManualRedactions(fileId).getResizeRedactions().stream().filter(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed())).findFirst().isPresent()){
|
||||
return MessageType.REANALYSE;
|
||||
}
|
||||
return MessageType.ANALYSE;
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusReprocess(String dossierId, String fileId, boolean priority) {
|
||||
|
||||
@ -451,6 +442,11 @@ public class FileStatusService {
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) {
|
||||
log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter());
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING);
|
||||
addToOcrQueue(dossierId, fileId, 2);
|
||||
}
|
||||
@ -493,19 +489,6 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
private void addToOcrQueue(String dossierId, String fileId, int priority) {
|
||||
|
||||
try {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new DocumentRequest(dossierId, fileId)), message -> {
|
||||
message.getMessageProperties().setPriority(priority);
|
||||
return message;
|
||||
});
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void updateLastOCRTime(String fileId) {
|
||||
|
||||
fileStatusPersistenceService.updateLastOCRTime(fileId, OffsetDateTime.now());
|
||||
@ -548,8 +531,7 @@ public class FileStatusService {
|
||||
|
||||
// wipe force redactions
|
||||
var forceRedactions = forceRedactionPersistenceService.findForceRedactions(fileId, false);
|
||||
forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId()
|
||||
.getAnnotationId(), now));
|
||||
forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
|
||||
|
||||
// wipe add manual redactions
|
||||
var addRedactions = addRedactionPersistenceService.findAddRedactions(fileId, false);
|
||||
@ -557,30 +539,108 @@ public class FileStatusService {
|
||||
|
||||
// wipe removeRedactions
|
||||
var removeRedactions = removeRedactionPersistenceService.findRemoveRedactions(fileId, false);
|
||||
removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId()
|
||||
.getAnnotationId(), now));
|
||||
removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
|
||||
|
||||
// wipe image recat
|
||||
var imageRecategorizations = imageRecategorizationPersistenceService.findRecategorizations(fileId, false);
|
||||
imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId()
|
||||
.getAnnotationId(), now));// wipe image recat
|
||||
imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));// wipe image recat
|
||||
|
||||
// wipe resize redactions
|
||||
var resizeRedactions = resizeRedactionPersistenceService.findResizeRedactions(fileId, false);
|
||||
resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId()
|
||||
.getAnnotationId(), now));
|
||||
resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
|
||||
|
||||
// wipe legal basis changes
|
||||
var legalBasisChanges = legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false);
|
||||
legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId()
|
||||
.getAnnotationId(), now));
|
||||
legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
|
||||
|
||||
fileStatusPersistenceService.updateHasComments(fileId, false);
|
||||
}
|
||||
|
||||
|
||||
private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(
|
||||
List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
|
||||
@Transactional
|
||||
public void setStatusAnalyse(String dossierId, String fileId, boolean priority) {
|
||||
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileStatus.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) {
|
||||
log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter());
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE);
|
||||
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false);
|
||||
}
|
||||
|
||||
|
||||
public int countSoftDeletedFiles(String dossierId) {
|
||||
|
||||
return fileStatusPersistenceService.countSoftDeletedFilesPerDossierId(dossierId);
|
||||
}
|
||||
|
||||
|
||||
protected void addToNerQueue(String dossierId, String fileId) {
|
||||
|
||||
setStatusNerAnalyzing(fileId);
|
||||
try {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(NerServiceRequest.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION)
|
||||
.responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION)
|
||||
.build()), message -> {
|
||||
message.getMessageProperties().setPriority(1);
|
||||
return message;
|
||||
});
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) {
|
||||
|
||||
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
|
||||
return MessageType.STRUCTURE_ANALYSE;
|
||||
}
|
||||
if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
|
||||
return MessageType.ANALYSE;
|
||||
}
|
||||
if (reanalyse) {
|
||||
if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) {
|
||||
return MessageType.ANALYSE;
|
||||
}
|
||||
return MessageType.REANALYSE;
|
||||
} else if (manualRedactionProviderService.getManualRedactions(fileId)
|
||||
.getResizeRedactions()
|
||||
.stream()
|
||||
.filter(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed()))
|
||||
.findFirst()
|
||||
.isPresent()) {
|
||||
return MessageType.REANALYSE;
|
||||
}
|
||||
return MessageType.ANALYSE;
|
||||
}
|
||||
|
||||
|
||||
private void addToOcrQueue(String dossierId, String fileId, int priority) {
|
||||
|
||||
try {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new DocumentRequest(dossierId, fileId)), message -> {
|
||||
message.getMessageProperties().setPriority(priority);
|
||||
return message;
|
||||
});
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
|
||||
|
||||
List<com.iqser.red.service.redaction.v1.model.FileAttribute> fileAttributeList = new ArrayList<>();
|
||||
|
||||
@ -611,44 +671,4 @@ public class FileStatusService {
|
||||
return fileStatusPersistenceService.hasChangesSince(dossierId, since);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusAnalyse(String dossierId, String fileId, boolean priority) {
|
||||
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileStatus.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE);
|
||||
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
|
||||
}
|
||||
|
||||
|
||||
protected void addToNerQueue(String dossierId, String fileId) {
|
||||
|
||||
setStatusNerAnalyzing(fileId);
|
||||
try {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(NerServiceRequest.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION)
|
||||
.responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION)
|
||||
.build()), message -> {
|
||||
message.getMessageProperties().setPriority(1);
|
||||
return message;
|
||||
});
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int countSoftDeletedFiles(String dossierId) {
|
||||
|
||||
return fileStatusPersistenceService.countSoftDeletedFilesPerDossierId(dossierId);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -30,4 +30,6 @@ public class FileManagementServiceSettings {
|
||||
|
||||
private boolean pdf2ImageServiceEnabled;
|
||||
|
||||
private int maxErrorRetries = 5;
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
databaseChangeLog:
|
||||
- changeSet:
|
||||
id: add-file-processing-error-counter-column
|
||||
author: philipp
|
||||
changes:
|
||||
- addColumn:
|
||||
columns:
|
||||
- column:
|
||||
name: processing_error_counter
|
||||
type: BIGINT
|
||||
tableName: file
|
||||
@ -77,3 +77,7 @@ databaseChangeLog:
|
||||
file: db/changelog/sql/31-watermark-configuration.sql
|
||||
- include:
|
||||
file: db/changelog/32-added-skipped-color-type-table.changelog.yaml
|
||||
- include:
|
||||
file: db/changelog/33-add-file-processing-error-counter-column.changelog.yaml
|
||||
- include:
|
||||
file: db/changelog/sql/33-set-file-processing-error-counter.sql
|
||||
|
||||
@ -0,0 +1 @@
|
||||
update file set processing_error_counter = 0 where processing_error_counter is null;
|
||||
Loading…
x
Reference in New Issue
Block a user