Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

reanalyse cleanup, imported flag for redaction-log

Browse Source
This commit is contained in:
Timo Bejan 2022-02-11 12:57:15 +02:00
parent b77e0fdcdf
commit 01569b6a0d
4 changed files with 34 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java
View File

@ -21,30 +21,25 @@ public interface ReanalysisResource {
@PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM) @PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM)
void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force); @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force);
@PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + BULK_REST_PATH) @PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + BULK_REST_PATH)
void reanalyzeFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds, void reanalyzeFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds,
@RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force); @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force);
@PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM) @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM)
void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId);
@RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser);
@PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_PARAM) @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_PARAM)
void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
@RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force); @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force);
@PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + BULK_REST_PATH) @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + BULK_REST_PATH)
void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
@RequestBody Set<String> fileIds); @RequestBody Set<String> fileIds);

47
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java
View File

@ -41,35 +41,33 @@ public class ReanalysisController implements ReanalysisResource {
@Override @Override
public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
var relevantFiles = getRelevantFiles(dossierId, triggeredByUser); var relevantFiles = getAllFilesForDossier(dossierId, true);
reanalyseFiles(dossierId, force, relevantFiles); reanalyseFiles(dossierId, force, relevantFiles);
} }
@Override @Override
public void reanalyzeFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds, public void reanalyzeFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds,
@RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
var relevantFiles = getRelevantFiles(dossierId, fileIds, triggeredByUser); var relevantFiles = getRelevantFiles(dossierId, fileIds);
reanalyseFiles(dossierId, force, relevantFiles); reanalyseFiles(dossierId, force, relevantFiles);
} }
public void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser) { public void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId) {
var relevantFiles = getRelevantFiles(dossierId, triggeredByUser); var relevantFiles = getAllFilesForDossier(dossierId, true);
relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId())); relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId()));
} }
public void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, @RequestBody Set<String> fileIds) { public void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds) {
var relevantFiles = getRelevantFiles(dossierId, fileIds, triggeredByUser); var relevantFiles = getRelevantFiles(dossierId, fileIds);
relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId())); relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId()));
} }
@ -81,7 +79,6 @@ public class ReanalysisController implements ReanalysisResource {
public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
@RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
dossierPersistenceService.getAndValidateDossier(dossierId); dossierPersistenceService.getAndValidateDossier(dossierId);
@ -96,36 +93,46 @@ public class ReanalysisController implements ReanalysisResource {
if (dossierFile.getLastOCRTime() != null) { if (dossierFile.getLastOCRTime() != null) {
throw new ConflictException("File already has been OCR processed"); throw new ConflictException("File already has been OCR processed");
} }
ocrFiles(dossierId,triggeredByUser, Sets.newHashSet(fileId)); ocrFiles(dossierId, Sets.newHashSet(fileId));
} }
} }
private List<FileEntity> getRelevantFiles(String dossierId, boolean triggeredByUser) { private List<FileEntity> getAllFilesForDossier(String dossierId, boolean filterOnlyValidFiles) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
return fileStatusService.getDossierStatus(dossier.getId())
.stream() if (filterOnlyValidFiles) {
.filter(fileStatus -> !fileStatus.getProcessingStatus().equals(ProcessingStatus.DELETED)) return fileStatusService.getDossierStatus(dossier.getId())
.filter(fileStatus -> !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) .stream()
.filter(fileStatus -> triggeredByUser || !fileStatus.isExcludedFromAutomaticAnalysis()) .filter(fileStatus -> !fileStatus.getProcessingStatus().equals(ProcessingStatus.DELETED))
.collect(Collectors.toList()); .filter(fileStatus -> !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED))
.collect(Collectors.toList());
} else {
return fileStatusService.getDossierStatus(dossier.getId());
}
} }
private List<FileEntity> getRelevantFiles(String dossierId, Collection<String> fileIds, boolean triggeredByUser) { private List<FileEntity> getRelevantFiles(String dossierId, Collection<String> fileIds) {
var relevantDossierFiles = getRelevantFiles(dossierId, triggeredByUser); var dossierFiles = getAllFilesForDossier(dossierId, false);
var relevantFiles = new ArrayList<FileEntity>(); var relevantFiles = new ArrayList<FileEntity>();
for (var fileId : fileIds) { for (var fileId : fileIds) {
var dossierFileOptional = relevantDossierFiles.stream().filter(f -> f.getId().equals(fileId)).findAny(); var dossierFileOptional = dossierFiles.stream().filter(f -> f.getId().equals(fileId)).findAny();
if (dossierFileOptional.isEmpty()) { if (dossierFileOptional.isEmpty()) {
throw new BadRequestException("Cannot reanalyse file from different dossier!"); throw new BadRequestException("Cannot reanalyse file from different dossier!");
} }
var dossierFile = dossierFileOptional.get(); var dossierFile = dossierFileOptional.get();
if (dossierFile.isSoftOrHardDeleted()) {
throw new BadRequestException("Cannot reanalyse deleted file!");
}
if (dossierFile.getWorkflowStatus() == WorkflowStatus.APPROVED) {
throw new BadRequestException("Cannot reanalyse approved file!");
}
relevantFiles.add(dossierFile); relevantFiles.add(dossierFile);

10
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java
View File

@ -42,28 +42,28 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier); var file = fileTesterAndProvider.testAndProvideFile(dossier);
reanalysisClient.ocrDossier(dossier.getId(),true); reanalysisClient.ocrDossier(dossier.getId());
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
resetProcessingStatus(file); resetProcessingStatus(file);
reanalysisClient.ocrFile(dossier.getId(), file.getId(),true, true); reanalysisClient.ocrFile(dossier.getId(), file.getId(),true);
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
resetProcessingStatus(file); resetProcessingStatus(file);
reanalysisClient.ocrFiles(dossier.getId(),true, Set.of(file.getId())); reanalysisClient.ocrFiles(dossier.getId(), Set.of(file.getId()));
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
resetProcessingStatus(file); resetProcessingStatus(file);
reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true,true); reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true);
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
resetProcessingStatus(file); resetProcessingStatus(file);
reanalysisClient.reanalyzeDossier(dossier.getId(), true,true); reanalysisClient.reanalyzeDossier(dossier.getId(), true);
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
resetProcessingStatus(file); resetProcessingStatus(file);

2
persistence-service-v1/pom.xml
View File

@ -25,7 +25,7 @@
</modules> </modules>
<properties> <properties>
<redaction-service.version>3.74.0</redaction-service.version> <redaction-service.version>3.76.0</redaction-service.version>
<search-service.version>2.18.0</search-service.version> <search-service.version>2.18.0</search-service.version>
<pdftron-redaction-service.version>3.31.0</pdftron-redaction-service.version> <pdftron-redaction-service.version>3.31.0</pdftron-redaction-service.version>
<redaction-report-service.version>3.19.0</redaction-report-service.version> <redaction-report-service.version>3.19.0</redaction-report-service.version>