Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

DM-357: always schedule reanalysis, when any manual redaction happens #74

Merged
kilian.schuettler1 merged 3 commits from DM-357 into master 2023-08-16 12:19:43 +02:00
Conversation 4 Commits 3 Files Changed 9 +25 -83
9 changed files with 25 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java
View File

@ -39,12 +39,7 @@ public class FileStatusProcessingUpdateService {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
switch (analyzeResult.getMessageType()) {
case SURROUNDING_TEXT:
fileStatusService.setStatusProcessed(analyzeResult.getFileId());
manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions());
break;
case ANALYSE:
case REANALYSE:
default:

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -83,7 +83,7 @@ public class FileStatusService {
var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(fileManagementServiceSettings.getMaxErrorRetries());
var convertedList = MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper());
log.info("Initial list of files for scheduler contains: {} files.", fileEntities.size());
log.debug("Initial list of files for scheduler contains: {} files.", fileEntities.size());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList());
}

76
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java
View File

@ -70,7 +70,6 @@ import lombok.extern.slf4j.Slf4j;
public class ManualRedactionService {
private final DossierPersistenceService dossierPersistenceService;
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final RemoveRedactionPersistenceService removeRedactionPersistenceService;
private final ForceRedactionPersistenceService forceRedactionPersistenceService;
@ -85,8 +84,6 @@ public class ManualRedactionService {
private final ManualRedactionProviderService manualRedactionProviderService;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
private final StopwordService stopwordService;
private final RabbitTemplate rabbitTemplate;
private final ObjectMapper objectMapper;
private final RedactionLogService redactionLogService;
private final DictionaryManagementService dictionaryManagementService;
private final HashFunction hashFunction = Hashing.murmur3_128();
@ -98,12 +95,13 @@ public class ManualRedactionService {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
// validate add to dossier template dictionaries
addRedactionRequests.forEach(request -> dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(request.getDossierTemplateTypeId(),
request.isAddToDictionary(),
request.isAddToAllDossiers()));
boolean actionPerformed = false;
for (var addRedactionRequest : addRedactionRequests) {
if (addRedactionRequest.isAddToDictionary()) {
validateDictionary(addRedactionRequest);
@ -118,7 +116,7 @@ public class ManualRedactionService {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
}
var addedToDictionary = handleAddToDictionary(fileId,
handleAddToDictionary(fileId,
annotationId,
addRedactionRequest.getDossierTemplateTypeId(),
addRedactionRequest.getValue(),
@ -129,28 +127,14 @@ public class ManualRedactionService {
dossierId,
addRedactionRequest.getDictionaryEntryType());
actionPerformed = actionPerformed || addedToDictionary;
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
actionPerformed = actionPerformed || addRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
if (actionPerformed) {
// in case of add to dict, there is no need to process surrounding text
reprocess(dossierId, fileId);
} else {
var manualTextRedactions = response.stream()
.map(r -> manualRedactionProviderService.getAddRedaction(fileId, r.getAnnotationId()))
.filter(m -> !m.isAddToDictionary() && !m.isRectangle())
.toList();
if (!manualTextRedactions.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
}
return response;
@ -244,20 +228,6 @@ public class ManualRedactionService {
}
private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING);
var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId).manualRedactions(manualRedactions).build();
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, analyseRequest, message -> {
message.getMessageProperties().setPriority(1);
return message;
});
}
private void removeFromDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) {
try {
@ -286,7 +256,7 @@ public class ManualRedactionService {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId);
var actionPerformed = false;
var requiresReAnalysis = false;
var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId);
//validate removing from dossier template dictionary
@ -314,18 +284,16 @@ public class ManualRedactionService {
if (removeRedactionRequest.getComment() != null) {
commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId();
}
if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) {
RedactionLogEntry redactionLogEntry = null;
boolean requiresAnalysis = false;
boolean matchingEntryFound;
try {
redactionLogEntry = getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId());
requiresAnalysis = redactionLogEntry.isHint();
getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId());
matchingEntryFound = true;
} catch (NotFoundException e) {
requiresAnalysis = false;
matchingEntryFound = false;
}
actionPerformed = actionPerformed || requiresAnalysis;
if (!requiresAnalysis && idRemoval.isApproved()) {
requiresReAnalysis = requiresReAnalysis || matchingEntryFound;
if (!matchingEntryFound && idRemoval.isApproved()) {
removeRedactionPersistenceService.markAsProcessed(idRemoval);
}
}
@ -343,13 +311,13 @@ public class ManualRedactionService {
removeRedactionPersistenceService.markAsProcessed(idRemoval);
}
actionPerformed = actionPerformed || removedFromDictionary;
requiresReAnalysis = requiresReAnalysis || removedFromDictionary;
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
}
if (actionPerformed) {
if (requiresReAnalysis) {
reprocess(dossierId, fileId);
}
@ -458,7 +426,7 @@ public class ManualRedactionService {
commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId();
}
actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(forceRedactionRequest.getStatus());
actionPerformed = actionPerformed || forceRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED);
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
@ -514,7 +482,7 @@ public class ManualRedactionService {
imageRecategorizationRequest.getUser()).getId();
}
actionPerformed = actionPerformed || !AnnotationStatus.REQUESTED.equals(imageRecategorizationRequest.getStatus());
actionPerformed = actionPerformed || imageRecategorizationRequest.getStatus().equals(AnnotationStatus.APPROVED);
response.add(ManualAddResponse.builder().annotationId(imageRecategorizationRequest.getAnnotationId()).commentId(commentId).build());
}
@ -745,7 +713,10 @@ public class ManualRedactionService {
addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeId);
resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(), resizeRedaction.getId().getAnnotationId(), resizeRedaction.getStatus(), typeIdsOfModifiedDictionaries);
resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(),
resizeRedaction.getId().getAnnotationId(),
resizeRedaction.getStatus(),
typeIdsOfModifiedDictionaries);
}
}
@ -779,15 +750,6 @@ public class ManualRedactionService {
}
@Transactional
public void updateSurroundingText(String fileId, ManualRedactions manualRedactions) {
// These are marked as processed once surrounding text is computed ( TBD if this is correct ? )
manualRedactions.getEntriesToAdd()
.forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
}
@Transactional
public void updateProcessedDate(String fileId, ManualRedactions manualRedactions) {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java
View File

@ -53,7 +53,7 @@ public class AutomaticAnalysisJob implements Job {
var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
if (redactionQueueInfo != null) {
log.info("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
tenant.getTenantId(),
MessagingConfiguration.REDACTION_QUEUE,
redactionQueueInfo.getMessageCount(),
@ -67,7 +67,7 @@ public class AutomaticAnalysisJob implements Job {
allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated));
var allStatusesIterator = allStatuses.iterator();
log.info("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size());
log.debug("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size());
var queuedFiles = 0;

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java
View File

@ -62,12 +62,6 @@ public class AddRedactionPersistenceService {
}
@Transactional
public void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter) {
manualRedactionRepository.updateSurroundingText(id, textBefore, textAfter);
}
public ManualRedactionEntryEntity findAddRedaction(String fileId, String annotationId) {

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java
View File

@ -38,11 +38,6 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
List<ManualRedactionEntryEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions);
@Modifying
@Query("update ManualRedactionEntryEntity m set m.textBefore = :textBefore, m.textAfter = :textAfter where m.id = :id")
void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter);
@Modifying
@Query("update ManualRedactionEntryEntity m set m.status = :newStatus, m.processedDate = :processedDate where m.id.fileId in :fileIds and m.value = :filterValue and m.addToDictionary = true and m.status = :filterStatus ")
void updateStatus(Set<String> fileIds, String filterValue, AnnotationStatus filterStatus, AnnotationStatus newStatus, OffsetDateTime processedDate);

5
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/MessageType.java
View File

@ -3,8 +3,5 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model;
public enum MessageType {
ANALYSE,
REANALYSE,
STRUCTURE_ANALYSE,
SURROUNDING_TEXT
REANALYSE
}

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java
View File

@ -10,6 +10,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
public class ManualAddResponse {
private String annotationId;
private Long commentId;

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java
View File

@ -12,7 +12,6 @@ public enum ProcessingStatus {
PROCESSED,
PROCESSING,
REPROCESS,
SURROUNDING_TEXT_PROCESSING,
UNPROCESSED,
FULL_PROCESSING,
PRE_PROCESSING_QUEUED,
@ -20,5 +19,4 @@ public enum ProcessingStatus {
PRE_PROCESSED,
FIGURE_DETECTION_ANALYZING,
TABLE_PARSING_ANALYZING
}