Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

DM-357: always schedule reanalysis, when any manual redaction happens

Browse Source 
* refactored code slightly
This commit is contained in:
Kilian Schuettler 2023-08-15 13:17:32 +02:00
parent f9af48af8c
commit 1f89d64d81
2 changed files with 20 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java
View File

@ -70,7 +70,6 @@ import lombok.extern.slf4j.Slf4j;
public class ManualRedactionService { public class ManualRedactionService {
private final DossierPersistenceService dossierPersistenceService; private final DossierPersistenceService dossierPersistenceService;
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final AddRedactionPersistenceService addRedactionPersistenceService; private final AddRedactionPersistenceService addRedactionPersistenceService;
private final RemoveRedactionPersistenceService removeRedactionPersistenceService; private final RemoveRedactionPersistenceService removeRedactionPersistenceService;
private final ForceRedactionPersistenceService forceRedactionPersistenceService; private final ForceRedactionPersistenceService forceRedactionPersistenceService;
@ -85,8 +84,6 @@ public class ManualRedactionService {
private final ManualRedactionProviderService manualRedactionProviderService; private final ManualRedactionProviderService manualRedactionProviderService;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService; private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
private final StopwordService stopwordService; private final StopwordService stopwordService;
private final RabbitTemplate rabbitTemplate;
private final ObjectMapper objectMapper;
private final RedactionLogService redactionLogService; private final RedactionLogService redactionLogService;
private final DictionaryManagementService dictionaryManagementService; private final DictionaryManagementService dictionaryManagementService;
private final HashFunction hashFunction = Hashing.murmur3_128(); private final HashFunction hashFunction = Hashing.murmur3_128();
@ -98,12 +95,13 @@ public class ManualRedactionService {
var response = new ArrayList<ManualAddResponse>(); var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId); dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
// validate add to dossier template dictionaries // validate add to dossier template dictionaries
addRedactionRequests.forEach(request -> dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(request.getDossierTemplateTypeId(), addRedactionRequests.forEach(request -> dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(request.getDossierTemplateTypeId(),
request.isAddToDictionary(), request.isAddToDictionary(),
request.isAddToAllDossiers())); request.isAddToAllDossiers()));
boolean actionPerformed = false;
for (var addRedactionRequest : addRedactionRequests) { for (var addRedactionRequest : addRedactionRequests) {
if (addRedactionRequest.isAddToDictionary()) { if (addRedactionRequest.isAddToDictionary()) {
validateDictionary(addRedactionRequest); validateDictionary(addRedactionRequest);
@ -118,7 +116,7 @@ public class ManualRedactionService {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId(); commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
} }
var addedToDictionary = handleAddToDictionary(fileId, handleAddToDictionary(fileId,
annotationId, annotationId,
addRedactionRequest.getDossierTemplateTypeId(), addRedactionRequest.getDossierTemplateTypeId(),
addRedactionRequest.getValue(), addRedactionRequest.getValue(),
@ -129,28 +127,14 @@ public class ManualRedactionService {
dossierId, dossierId,
addRedactionRequest.getDictionaryEntryType()); addRedactionRequest.getDictionaryEntryType());
actionPerformed = actionPerformed || addedToDictionary;
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
actionPerformed = actionPerformed || addRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED);
} }
analysisFlagsCalculationService.calculateFlags(dossierId, fileId); analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
if (actionPerformed) { if (actionPerformed) {
// in case of add to dict, there is no need to process surrounding text
reprocess(dossierId, fileId); reprocess(dossierId, fileId);
} else {
var manualTextRedactions = response.stream()
.map(r -> manualRedactionProviderService.getAddRedaction(fileId, r.getAnnotationId()))
.filter(m -> !m.isAddToDictionary() && !m.isRectangle())
.toList();
if (!manualTextRedactions.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
} }
return response; return response;
@ -244,20 +228,6 @@ public class ManualRedactionService {
} }
private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING);
var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId).manualRedactions(manualRedactions).build();
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, analyseRequest, message -> {
message.getMessageProperties().setPriority(1);
return message;
});
}
private void removeFromDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) { private void removeFromDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) {
try { try {
@ -286,7 +256,7 @@ public class ManualRedactionService {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId); RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId);
var actionPerformed = false; var requiresReAnalysis = false;
var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId); var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId);
//validate removing from dossier template dictionary //validate removing from dossier template dictionary
@ -314,18 +284,16 @@ public class ManualRedactionService {
if (removeRedactionRequest.getComment() != null) { if (removeRedactionRequest.getComment() != null) {
commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId(); commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId();
} }
if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) { if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) {
RedactionLogEntry redactionLogEntry = null; boolean matchingEntryFound;
boolean requiresAnalysis = false;
try { try {
redactionLogEntry = getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()); getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId());
requiresAnalysis = redactionLogEntry.isHint(); matchingEntryFound = true;
} catch (NotFoundException e) { } catch (NotFoundException e) {
requiresAnalysis = false; matchingEntryFound = false;
} }
actionPerformed = actionPerformed || requiresAnalysis; requiresReAnalysis = requiresReAnalysis || matchingEntryFound;
if (!requiresAnalysis && idRemoval.isApproved()) { if (!matchingEntryFound && idRemoval.isApproved()) {
removeRedactionPersistenceService.markAsProcessed(idRemoval); removeRedactionPersistenceService.markAsProcessed(idRemoval);
} }
} }
@ -343,13 +311,13 @@ public class ManualRedactionService {
removeRedactionPersistenceService.markAsProcessed(idRemoval); removeRedactionPersistenceService.markAsProcessed(idRemoval);
} }
actionPerformed = actionPerformed || removedFromDictionary; requiresReAnalysis = requiresReAnalysis || removedFromDictionary;
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build()); response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
} }
} }
if (actionPerformed) { if (requiresReAnalysis) {
reprocess(dossierId, fileId); reprocess(dossierId, fileId);
} }
@ -458,7 +426,7 @@ public class ManualRedactionService {
commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId(); commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId();
} }
actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(forceRedactionRequest.getStatus()); actionPerformed = actionPerformed || forceRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED);
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build()); response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
} }
@ -514,7 +482,7 @@ public class ManualRedactionService {
imageRecategorizationRequest.getUser()).getId(); imageRecategorizationRequest.getUser()).getId();
} }
actionPerformed = actionPerformed || !AnnotationStatus.REQUESTED.equals(imageRecategorizationRequest.getStatus()); actionPerformed = actionPerformed || imageRecategorizationRequest.getStatus().equals(AnnotationStatus.APPROVED);
response.add(ManualAddResponse.builder().annotationId(imageRecategorizationRequest.getAnnotationId()).commentId(commentId).build()); response.add(ManualAddResponse.builder().annotationId(imageRecategorizationRequest.getAnnotationId()).commentId(commentId).build());
} }
@ -745,7 +713,10 @@ public class ManualRedactionService {
addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType); addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeId); typeIdsOfModifiedDictionaries.add(typeId);
resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(), resizeRedaction.getId().getAnnotationId(), resizeRedaction.getStatus(), typeIdsOfModifiedDictionaries); resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(),
resizeRedaction.getId().getAnnotationId(),
resizeRedaction.getStatus(),
typeIdsOfModifiedDictionaries);
} }
} }

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java
View File

@ -10,6 +10,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor @AllArgsConstructor
@NoArgsConstructor @NoArgsConstructor
public class ManualAddResponse { public class ManualAddResponse {
private String annotationId; private String annotationId;
private Long commentId; private Long commentId;