Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9864: prevent ocr being queued multiple times

Browse Source
This commit is contained in:
Kilian Schüttler 2024-08-29 12:14:20 +02:00
parent 7a209664cb
commit 5188ab416e
4 changed files with 31 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts
View File

@ -45,6 +45,8 @@ tasks.named<Test>("test") {
reports {
junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
}
minHeapSize = "512m"
maxHeapSize = "2048m"
}
tasks.test {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java
View File

@ -124,14 +124,14 @@ public class AnalysisFlagsCalculationService {
}
log.info("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime);
log.debug("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime);
if (file.isHasRedactions() == hasRedactions
&& file.isHasHints() == hasHints
&& file.isHasImages() == hasImages
&& file.isHasSuggestions() == hasSuggestions
&& file.isHasAnnotationComments() == hasComments
&& file.isHasUpdates() == hasUpdates) {
log.info("Nothing Changed for file: {}", fileId);
log.debug("Nothing Changed for file: {}", fileId);
} else {
fileStatusPersistenceService.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates);
}

18
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -266,7 +266,14 @@ public class FileStatusService {
fileModel = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
var dossierTemplate = dossier.getDossierTemplate();
if (dossierTemplate.isOcrByDefault() && fileModel.getOcrEndTime() == null && !fileModel.isSoftOrHardDeleted()) {
if (dossierTemplate.isOcrByDefault()//
&& fileModel.getOcrEndTime() == null //
&& !fileModel.isSoftOrHardDeleted()) {
if (ocrIsRunning(fileModel)) {
return;
}
log.info("Add file: {} from dossier {} to OCR queue", fileId, dossierId);
setStatusOcrQueued(dossierId, fileId);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
@ -304,6 +311,7 @@ public class FileStatusService {
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
if (settings.isAzureNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.AZURE_NER_ENTITIES)) {
log.info("Add file: {} from dossier {} to AZURE NER queue", fileId, dossierId);
addToAzureNerQueue(dossierId, fileId);
@ -357,6 +365,13 @@ public class FileStatusService {
}
private static boolean ocrIsRunning(FileModel fileModel) {
return fileModel.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING_QUEUED) //
|| fileModel.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING);
}
private void sendAnalysisEvent(String dossierId, String fileId, FileEntity fileEntity) {
if (fileEntity.getProcessingStatus().equals(ProcessingStatus.UNPROCESSED)
@ -823,6 +838,7 @@ public class FileStatusService {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.LLM_NER_ENTITIES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.AZURE_NER_ENTITIES);
}
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false);

20
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java
View File

@ -152,16 +152,16 @@ public class ReanalysisRequiredStatusService {
if (reanalysisRequired || fullAnalysisRequired) {
log.info(buildMessage(fileStatus,
rulesVersionMatches,
dossierTemplateVersions,
componentRulesVersionMatches,
dictionaryVersionMatches,
legalBasisVersionMatches,
dossierDictionaryVersionMatches,
dossierDictionaryVersion,
mappingVersionAllMatch,
componentMappingVersions));
log.debug(buildMessage(fileStatus,
rulesVersionMatches,
dossierTemplateVersions,
componentRulesVersionMatches,
dictionaryVersionMatches,
legalBasisVersionMatches,
dossierDictionaryVersionMatches,
dossierDictionaryVersion,
mappingVersionAllMatch,
componentMappingVersions));
}
return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired);