Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-10127' into 'master'

Browse Source 
RED-10127: enable ocr rerun via support controller

Closes RED-10127

See merge request redactmanager/persistence-service!787
This commit is contained in:
Kilian Schüttler 2024-10-23 10:35:12 +02:00
commit f3823a9acd
16 changed files with 102 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java vendored
View File

@ -272,7 +272,7 @@ public class FileManagementController implements FileManagementResource {
// if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) {
// reanalysisService.ocrFile(dossierId, fileId, true);
// } else {
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
// }
}

11
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java vendored
View File

@ -60,18 +60,20 @@ public class SupportController implements SupportResource {
@Override
public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
@RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr) {
reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis);
reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis, runOcr);
}
@Override
public void reanalyzeErrorFilesBulkForDossier(@PathVariable(DOSSIER_ID) String dossierId,
@RequestBody List<String> fileIds,
@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
@RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr) {
reanalysisService.reanalyzeGivenErrorFilesInDossier(dossierId, new HashSet<>(fileIds), repeatStructureAnalysis);
reanalysisService.reanalyzeGivenErrorFilesInDossier(dossierId, new HashSet<>(fileIds), repeatStructureAnalysis, runOcr);
}
@ -156,6 +158,7 @@ public class SupportController implements SupportResource {
return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
}
@Override
@PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
public ImportResponse importDataset(MultipartFile file) {

12
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java vendored
View File

@ -51,6 +51,7 @@ public interface SupportResource {
String FALSE = "false";
String FULL_REANALYSIS_PARAM = "fullReanalysis";
String RUN_OCR_PARAM = "runOcr";
String EXPORT = "/export";
String IMPORT = "/import";
@ -80,7 +81,8 @@ public interface SupportResource {
@PostMapping(value = ERROR_REANALYSIS_REST_PATH)
@Operation(summary = "Reanalyze all files in error state.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
@RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr);
@PostMapping(value = ERROR_REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE + BULK_REST_PATH)
@ -88,7 +90,8 @@ public interface SupportResource {
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
void reanalyzeErrorFilesBulkForDossier(@PathVariable(DOSSIER_ID) String dossierId,
@RequestBody List<String> fileIds,
@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
@RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr);
@ResponseStatus(value = HttpStatus.OK)
@ -146,9 +149,7 @@ public interface SupportResource {
@ResponseStatus(value = HttpStatus.OK)
@ResponseBody
@PostMapping(value = DATASET_EXCHANGE
+ EXPORT
+ DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@PostMapping(value = DATASET_EXCHANGE + EXPORT + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
## Export Preview Files Endpoint
@ -166,6 +167,7 @@ public interface SupportResource {
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

4
persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java
View File

@ -37,7 +37,7 @@ public class AdminInterfaceController {
fileManagementStorageService.deleteAllObjects(dossierId, fileId);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
}
@ -136,7 +136,7 @@ public class AdminInterfaceController {
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
}
}

5
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -38,10 +38,7 @@ dependencies {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight.service:ocr-service-api:4.24.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight:azure-ocr-service-api:0.13.0")
implementation("com.knecon.fforesight:llm-service-api:1.17.0")
api("com.knecon.fforesight:jobs-commons:0.10.0")
api("com.iqser.red.commons:storage-commons:2.50.0")

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java
View File

@ -78,8 +78,8 @@ public class DatasetExchangeService {
var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId);
String downloadFilename = template.getName() + "_PREVIEW_FILES.zip";
String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES");
List<FileEntity> files = fileStatusPersistenceService.getStatusesForDossierTemplate(dossierTemplateId);
List<DossierEntity> dossiers = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
List<FileEntity> files = fileStatusPersistenceService.getActiveFilesForDossierTemplate(dossierTemplateId);
List<DossierEntity> dossiers = dossierPersistenceService.findAllActiveDossiersForDossierTemplateId(dossierTemplateId);
if (dossiers.isEmpty()) {
throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java
View File

@ -58,7 +58,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration {
files.forEach(file -> {
if (file.getHardDeletedTime() == null && !file.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
log.info("Set full reanalyse for file {}", file.getId());
fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true);
fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true, false);
log.info("Finished migration of file {}", file.getId());
}
});

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
import org.apache.commons.lang3.StringUtils;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@ -160,7 +159,7 @@ public class FileStatusProcessingUpdateService {
retryTemplate.execute(retryContext -> {
log.info("OCR Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true);
fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true, false);
return null;
});

46
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -5,13 +5,14 @@ import static com.iqser.red.service.persistence.management.v1.processor.configur
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
@ -76,6 +77,7 @@ import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverte
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.llm.service.LlmNerMessage;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -144,6 +146,7 @@ public class FileStatusService {
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
}
public List<DossierStatsFileProjection> getDossierStatsFiles(String dossierId) {
return fileStatusPersistenceService.getFilesForDossierStats(dossierId);
@ -718,13 +721,26 @@ public class FileStatusService {
public void addToOcrQueue(String dossierId, String fileId, int priority) {
var removeWatermark = dossierTemplatePersistenceService.getDossierTemplate(dossierPersistenceService.getDossierTemplateId(dossierId)).isRemoveWatermark();
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_REQUEST_EXCHANGE,
TenantContext.getTenantId(),
new DocumentRequest(dossierId, fileId, removeWatermark),
message -> {
message.getMessageProperties().setPriority(priority);
return message;
});
Set<AzureOcrFeature> features = new HashSet<>();
if (removeWatermark) {
features.add(AzureOcrFeature.REMOVE_WATERMARKS);
}
if (applicationType.equals("DocuMine")) {
features.add(AzureOcrFeature.ROTATION_CORRECTION);
features.add(AzureOcrFeature.FONT_STYLE_DETECTION);
}
var request = DocumentRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.removeWatermark(features.contains(AzureOcrFeature.REMOVE_WATERMARKS)) // needed for legacy OCR-services
.features(features)
.build();
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_REQUEST_EXCHANGE, TenantContext.getTenantId(), request, message -> {
message.getMessageProperties().setPriority(priority);
return message;
});
}
@ -893,13 +909,13 @@ public class FileStatusService {
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
if (!excluded) {
setStatusFullReprocess(dossierId, fileId, false, true);
setStatusFullReprocess(dossierId, fileId, false, true, false);
}
}
@Transactional
public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) {
public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis, boolean runOcr) {
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
@ -908,11 +924,17 @@ public class FileStatusService {
return;
}
if (requiresStructureAnalysis) {
if (requiresStructureAnalysis || runOcr) {
log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId);
}
if (runOcr) {
fileStatusPersistenceService.resetOcrStartAndEndDate(fileId);
setStatusOcrQueued(dossierId, fileId);
return;
}
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), AnalysisType.DEFAULT);
}
@ -925,7 +947,7 @@ public class FileStatusService {
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
if (!excludedFromAutomaticAnalysis) {
setStatusFullReprocess(dossierId, fileId, false, true);
setStatusFullReprocess(dossierId, fileId, false, true, false);
}
}

24
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java
View File

@ -48,10 +48,10 @@ public class ReanalysisService {
private final Predicate<FileModel> errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR);
public void reanalyzeAllRelevantErrorFiles(boolean repeatStructureAnalysis) {
public void reanalyzeAllRelevantErrorFiles(boolean repeatStructureAnalysis, boolean runOcr) {
var errorFiles = fileStatusService.getAllRelevantErrorFiles();
analyseFiles(repeatStructureAnalysis, errorFiles);
analyseFiles(repeatStructureAnalysis, errorFiles, runOcr);
}
@ -88,7 +88,7 @@ public class ReanalysisService {
if (force) {
filesToReanalyse.forEach(file -> {
if (file.isFullAnalysisRequired()) {
fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false);
fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false, false);
} else {
fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, true);
}
@ -98,7 +98,7 @@ public class ReanalysisService {
.filter(FileModel::isReanalysisRequired)
.forEach(file -> {
if (file.isFullAnalysisRequired()) {
fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false);
fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false, false);
} else {
fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, true);
}
@ -107,9 +107,9 @@ public class ReanalysisService {
}
private void analyseFiles(boolean repeatStructureAnalysis, List<FileModel> filesToReanalyse) {
private void analyseFiles(boolean repeatStructureAnalysis, List<FileModel> filesToReanalyse, boolean runOcr) {
filesToReanalyse.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, repeatStructureAnalysis));
filesToReanalyse.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, repeatStructureAnalysis, runOcr));
}
@ -120,9 +120,9 @@ public class ReanalysisService {
}
public void reanalyzeGivenErrorFilesInDossier(String dossierId, Set<String> fileIds, boolean repeatStructureAnalysis) {
public void reanalyzeGivenErrorFilesInDossier(String dossierId, Set<String> fileIds, boolean repeatStructureAnalysis, boolean runOcr) {
analyseFiles(repeatStructureAnalysis, fileIds.isEmpty() ? getAllFilesForDossier(dossierId, errorFilesFilter) : getRelevantErrorFiles(dossierId, fileIds));
analyseFiles(repeatStructureAnalysis, fileIds.isEmpty() ? getAllFilesForDossier(dossierId, errorFilesFilter) : getRelevantErrorFiles(dossierId, fileIds), runOcr);
}
@ -231,7 +231,7 @@ public class ReanalysisService {
} catch (Exception e) {
throw new BadRequestException("File Not Valid PDF or number of pages do not match");
}
fileStatusService.setStatusFullReprocess(documentRequest.getDossierId(), documentRequest.getFileId(), true, false);
fileStatusService.setStatusFullReprocess(documentRequest.getDossierId(), documentRequest.getFileId(), true, false, false);
}
@ -245,7 +245,7 @@ public class ReanalysisService {
deleteImportedRedactionsRequest.getFileId(),
FileType.IMPORTED_REDACTIONS,
importedRedactions);
fileStatusService.setStatusFullReprocess(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), true, false);
fileStatusService.setStatusFullReprocess(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), true, false, false);
}
@ -263,7 +263,7 @@ public class ReanalysisService {
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) {
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false);
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false, false);
}
}
@ -281,7 +281,7 @@ public class ReanalysisService {
List<FileModel> rejectedFiles = filterInvalidFiles(files);
files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis()));
files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis(), reanalysisSettings.runOcr()));
return rejectedFiles;
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java
View File

@ -125,7 +125,7 @@ public class AutomaticAnalysisJob implements Job {
if (file.isFullAnalysisRequired()) {
log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getId());
fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false);
fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false, false);
} else if (file.isReanalysisRequired()) {
log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getId());
fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), false);

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java
View File

@ -194,6 +194,11 @@ public class DossierPersistenceService {
return dossierRepository.findByDossierTemplateId(dossierTemplateId);
}
public List<DossierEntity> findAllActiveDossiersForDossierTemplateId(String dossierTemplateId) {
return dossierRepository.findActiveByDossierTemplateId(dossierTemplateId);
}
public Set<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId) {

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -414,6 +414,16 @@ public class FileStatusPersistenceService {
.toList();
}
public List<FileEntity> getActiveFilesForDossierTemplate(String dossierTemplateId) {
return dossierService.findAllActiveDossiersForDossierTemplateId(dossierTemplateId)
.stream()
.map(dossier -> fileRepository.findByDossierId(dossier.getId()))
.flatMap(List::stream)
.filter(f -> !f.isSoftOrHardDeleted())
.toList();
}
public List<FileEntity> getStatusesAddedBefore(OffsetDateTime end) {

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java
View File

@ -115,4 +115,7 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
@Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL")
Optional<String> findDossierNameById(@Param("dossierId") String dossierId);
@Query("select d from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null")
List<DossierEntity> findActiveByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
}

26
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java
View File

@ -65,35 +65,35 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
var dossier = dossierTesterAndProvider.provideTestDossier();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
var e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
var e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false));
assertThat(e.status()).isEqualTo(409);
setProcessingStatusToError(dossier, file);
supportClient.reanalyzeAllRelevantErrorFiles(true);
supportClient.reanalyzeAllRelevantErrorFiles(true, false);
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
setProcessingStatusToError(dossier, file);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(), true);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(), true, false);
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
setProcessingStatusToError(dossier, file);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false);
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
setProcessingStatusToError(dossier, file);
setFileToStatusDeleted(file);
e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false));
assertThat(e.status()).isEqualTo(400);
setFileToStatusHardDeleted(file);
e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false));
assertThat(e.status()).isEqualTo(400);
}
@ -177,7 +177,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
dossierClient.deleteDossier(dossier1.getId());
supportClient.reanalyzeAllRelevantErrorFiles(false);
supportClient.reanalyzeAllRelevantErrorFiles(false, false);
FeignException feignException = assertThrows(FeignException.class, () -> fileClient.getFileStatus(dossier1.getId(), file1.getId()));
assertEquals(feignException.status(), 404);
FileStatus fileStatus = FileStatusMapper.toFileStatus(fileStatusManagementService.getFileStatus(file1.getId()));
@ -187,11 +187,11 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
setProcessingStatusToError(dossier2, file2);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true, false);
loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true));
feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true, false));
assertEquals(feignException.status(), 404);
setProcessingStatusToError(dossier2, file2);
@ -200,6 +200,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()),
Collections.emptySet(),
true,
false,
new FileStatusFilter(null, null, true, true)));
loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
@ -224,7 +225,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
dossierClient.archiveDossiers(Collections.singleton(dossier1.getId()));
supportClient.reanalyzeAllRelevantErrorFiles(false);
supportClient.reanalyzeAllRelevantErrorFiles(false, false);
FileStatus fileStatus = fileClient.getFileStatus(dossier1.getId(), file1.getId());
assertThat(fileStatus.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
var loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
@ -232,11 +233,11 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
setProcessingStatusToError(dossier2, file2);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true, false);
loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
FeignException feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true));
FeignException feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true, false));
assertEquals(feignException.status(), 404);
setProcessingStatusToError(dossier2, file2);
@ -245,6 +246,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()),
Collections.emptySet(),
true,
false,
null));
loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java
View File

@ -9,6 +9,7 @@ public record ReanalysisSettings(
@Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set<String> dossierIds,
@Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set<String> fileIds,
@Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis,
@Schema(description = "If set to true, ocr will be repeated and therefore also layout parsing and named entity recognition.", defaultValue = "false") boolean runOcr,
@Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "{}") FileStatusFilter fileStatusFilter
) {