Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #514: RED-4861: Do no analysis on request resize/removeHint

Browse Source 
Merge in RED/persistence-service from RED-4861-2 to master

* commit '13a96d4cade637cbd705361dc6582fc7af427e33':
  RED-4861: Do no analysis on request resize/removeHint
This commit is contained in:
Dominique Eiflaender 2022-08-09 09:13:41 +02:00
commit 5e53398685
3 changed files with 67 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java
View File

@ -43,7 +43,7 @@ public class FileStatusProcessingUpdateService {
fileStatusService.setStatusAnalyse(dossierId, fileId, false);
//TODO This might be also priority depending on what was the pervious call.
fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null);
fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null, false);
}
break;

45
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -128,7 +128,7 @@ public class FileStatusService {
long fileSize) {
fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize);
addToAnalysisQueue(dossierId, fileId, false, Set.of());
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) {
addToPdf2ImageQueue(dossierId, fileId, Set.of());
@ -181,7 +181,7 @@ public class FileStatusService {
@Transactional
public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse,
boolean triggeredManually) {
boolean triggeredManually, boolean manualRedactionReanalyse) {
log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
@ -198,7 +198,7 @@ public class FileStatusService {
}
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS);
addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse);
addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse, manualRedactionReanalyse);
}
@ -231,7 +231,7 @@ public class FileStatusService {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
}
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false);
}
@ -253,14 +253,19 @@ public class FileStatusService {
@Transactional
public void setStatusReprocess(String dossierId, String fileId, boolean priority) {
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false);
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false, false);
}
@Transactional
public void setStatusReprocessForManual(String dossierId, String fileId, boolean priority) {
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false, true);
}
@Transactional
public void setStatusReprocess(String dossierId, String fileId, boolean priority, boolean triggeredManually) {
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually);
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually, priority);
}
@ -268,7 +273,7 @@ public class FileStatusService {
public void createStatus(String dossierId, String fileId, String uploader, String filename) {
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader);
addToAnalysisQueue(dossierId, fileId, false, Set.of());
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
}
@ -450,7 +455,7 @@ public class FileStatusService {
viewedPagesPersistenceService.deleteForFile(fileId);
addToAnalysisQueue(dossierId, fileId, false, Set.of());
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
}
@ -511,7 +516,7 @@ public class FileStatusService {
}
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE);
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false);
}
@ -529,7 +534,7 @@ public class FileStatusService {
@Transactional
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority,
Set<Integer> sectionsToReanalyse) {
Set<Integer> sectionsToReanalyse, boolean manualRedactionReanalyse) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
@ -547,7 +552,7 @@ public class FileStatusService {
var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
boolean reanalyse = fileModel.isReanalysisRequired();
boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse;
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
@ -630,29 +635,11 @@ public class FileStatusService {
return MessageType.ANALYSE;
}
return MessageType.REANALYSE;
} else if (hasUnprocessedManualRedactions(fileModel)) {
return MessageType.REANALYSE;
}
return MessageType.ANALYSE;
}
private boolean hasUnprocessedManualRedactions(FileModel fileModel) {
return fileModel.getLastProcessed() != null && (manualRedactionProviderService.getManualRedactions(fileModel.getId())
.getResizeRedactions()
.stream()
.anyMatch(manual -> manual.getRequestDate()
.isAfter(fileModel.getLastProcessed())) || manualRedactionProviderService.getManualRedactions(fileModel.getId())
.getForceRedactions()
.stream()
.anyMatch(manual -> manual.getRequestDate()
.isAfter(fileModel.getLastProcessed())) || manualRedactionProviderService.getManualRedactions(fileModel.getId())
.getIdsToRemove()
.stream()
.anyMatch(manual -> manual.getRequestDate().isAfter(fileModel.getLastProcessed())));
}
private void addToOcrQueue(String dossierId, String fileId, int priority) {

69
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -1,5 +1,23 @@
package com.iqser.red.service.peristence.v1.server.service;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.hash.HashFunction;
@ -7,7 +25,14 @@ import com.google.common.hash.Hashing;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@ -16,8 +41,23 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.CommentRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
@ -26,21 +66,10 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.MessageType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
@Slf4j
@Service
@ -174,7 +203,7 @@ public class ManualRedactionService {
commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId();
}
if (!removeRedactionRequest.isRemoveFromDictionary()) {
if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) {
Optional<RedactionLogEntry> redactionLogEntryOptional = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(removeRedactionRequest.getAnnotationId()))
@ -463,8 +492,10 @@ public class ManualRedactionService {
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
reprocess(dossierId, fileId);
if(resizeRedactionRequests.stream().anyMatch(resize -> AnnotationStatus.APPROVED.equals(resize.getStatus()))) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
@ -726,7 +757,7 @@ public class ManualRedactionService {
private void reprocess(String dossierId, String fileId) {
fileStatusService.setStatusReprocess(dossierId, fileId, true);
fileStatusService.setStatusReprocessForManual(dossierId, fileId, true);
}