RED-2439: Enabled to find surrounding text for manual redactions via queue

This commit is contained in:
deiflaender 2021-12-13 13:26:45 +01:00
parent 83296cc093
commit afeee3811c
8 changed files with 141 additions and 55 deletions

View File

@ -43,6 +43,12 @@ public class ResizeRedactionPersistenceService {
}
@Transactional
public void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter){
resizeRedactionRepository.updateSurroundingText(id, textBefore, textAfter);
}
@Transactional
public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) {

View File

@ -32,4 +32,8 @@ public interface ResizeRedactionRepository extends JpaRepository<ManualResizeRed
@Query("select mrd from ManualResizeRedactionEntity mrd where mrd.id.fileId = :fileId and (:includeDeletions = true or mrd.softDeletedTime is null)")
List<ManualResizeRedactionEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions);
@Modifying
@Query("update ManualResizeRedactionEntity m set m.textBefore = :textBefore, m.textAfter = :textAfter where m.id = :id")
void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter);
}

View File

@ -49,7 +49,7 @@ public class MessagingConfiguration {
return QueueBuilder.durable(OCR_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", OCR_DLQ)
.withArgument("x-max-priority", 2)
.withArgument("x-max-priority", 2) // Higher value is higher priority.
.maxPriority(2)
.build();
}

View File

@ -1,13 +1,13 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.web.bind.annotation.RestController;
import java.util.Objects;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RestController
@ -19,24 +19,36 @@ public class FileStatusProcessingUpdateService {
private final IndexingService indexingService;
private final DossierPersistenceService dossierPersistenceService;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
private final ManualRedactionService manualRedactionService;
public void analysisSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
retryTemplate.execute(retryContext -> {
log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext
.getRetryCount());
fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult);
return null;
});
switch (analyzeResult.getMessageType()) {
if (!analyzeResult.isWasReanalyzed()) {
indexingService.addToIndexingQueue(dossier.getDossierTemplateId(), dossierId, fileId, 2);
case SURROUNDING_TEXT:
fileStatusService.setStatusProcessed(analyzeResult.getFileId());
manualRedactionService.updateManualRedactions(fileId, analyzeResult.getManualRedactions());
break;
case REANALYSE:
case FULL_ANALYSE:
default:
retryTemplate.execute(retryContext -> {
log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext
.getRetryCount());
fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult);
return null;
});
if (!analyzeResult.isWasReanalyzed()) {
indexingService.addToIndexingQueue(dossier.getDossierTemplateId(), dossierId, fileId, 2);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}

View File

@ -15,6 +15,8 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.MessageType;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -73,6 +75,12 @@ public class FileStatusService {
}
public void setStatusProcessed(String fileId) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSED);
}
public void setStatusProcessing(String fileId) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING);
@ -234,7 +242,12 @@ public class FileStatusService {
var fileAttributes = fileStatus.getFileAttributes();
boolean reanalyse = !fileStatus.getProcessingStatus().equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
.equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
.isAfter(fileStatus.getLastFileAttributeChange()));
var analyseRequest = AnalyzeRequest.builder()
.messageType(reanalyse ? MessageType.REANALYSE : MessageType.FULL_ANALYSE)
.dossierId(dossierId)
.sectionsToReanalyse(sectionsToReanalyse)
.fileId(fileId)
@ -242,13 +255,9 @@ public class FileStatusService {
.dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileStatus.getLastProcessed())
.fileAttributes(convert(fileAttributes))
.excludedPages(fileStatus.getExcludedPages())
.build();
analyseRequest.setReanalyseOnlyIfPossible(!fileStatus.getProcessingStatus().equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
.equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
.isAfter(fileStatus.getLastFileAttributeChange())));
analyseRequest.setExcludedPages(fileStatus.getExcludedPages());
setStatusProcessing(fileId);

View File

@ -1,11 +1,15 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper;
import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
@ -14,11 +18,17 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ManualRedactionRepository;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.MessageType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
@ -51,7 +61,8 @@ public class ManualRedactionService {
private final ManualRedactionProviderService manualRedactionProviderService;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
private final StopwordService stopwordService;
private final RedactionClient redactionClient;
private final RabbitTemplate rabbitTemplate;
private final ObjectMapper objectMapper;
private final HashFunction hashFunction = Hashing.murmur3_128();
@ -75,15 +86,15 @@ public class ManualRedactionService {
}
}
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8)
.toString();
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
OffsetDateTime now = OffsetDateTime.now();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
Long commentId = null;
if (addRedactionRequest.getComment() != null) {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser())
.getId();
}
handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest
@ -99,17 +110,11 @@ public class ManualRedactionService {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
// TODO This is just a proof of concept.
var loaded = convert(getAddRedaction(fileId, annotationId), ManualRedactionEntry.class, new ManualRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder()
.entriesToAdd(Set.of(loaded))
.build();
var withSurroundingText = redactionClient.addSurroundingText(dossierId, fileId, manualRedactions);
if(withSurroundingText != null) {
withSurroundingText.getEntriesToAdd().forEach(e -> {
addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(annotationId, fileId), e.getTextBefore(), e
.getTextAfter());
});
if (!addRedactionRequest.isAddToDictionary() && !addRedactionRequest.isAddToDossierDictionary() && !addRedactionRequest
.isRectangle()) {
var loaded = convert(getAddRedaction(fileId, annotationId), ManualRedactionEntry.class, new ManualRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(Set.of(loaded)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build();
@ -131,8 +136,8 @@ public class ManualRedactionService {
.getUser()).getId();
}
handleRemoveFromDictionary(dossierId, fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(),
removeRedactionRequest.isRemoveFromDictionary(), false);
handleRemoveFromDictionary(dossierId, fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), removeRedactionRequest
.isRemoveFromDictionary(), false);
if (removeRedactionRequest.getStatus().equals(AnnotationStatus.REQUESTED)) {
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, now, true);
@ -252,8 +257,9 @@ public class ManualRedactionService {
return createdComment;
}
public ManualAddResponse addResizeRedaction(String dossierId, String fileId,
ResizeRedactionRequest resizeRedactionRequest) {
ResizeRedactionRequest resizeRedactionRequest) {
dossierPersistenceService.getAndValidateDossier(dossierId);
@ -275,6 +281,10 @@ public class ManualRedactionService {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
var loaded = convert(getResizeRedaction(fileId, resizeRedactionRequest.getAnnotationId()), ManualResizeRedaction.class, new ManualResizeRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of(loaded)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
return ManualAddResponse.builder()
.annotationId(resizeRedactionRequest.getAnnotationId())
.commentId(commentId)
@ -329,8 +339,8 @@ public class ManualRedactionService {
var addRedaction = getAddRedaction(fileId, annotationId);
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
handleAddToDictionary(fileId, annotationId, addRedaction.getTypeId(), addRedaction.getValue(), addRedaction.getStatus(),
addRedaction.isAddToDictionary(), true, dossier.getId());
handleAddToDictionary(fileId, annotationId, addRedaction.getTypeId(), addRedaction.getValue(), addRedaction.getStatus(), addRedaction
.isAddToDictionary(), true, dossier.getId());
addRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
@ -353,8 +363,7 @@ public class ManualRedactionService {
var removeRedaction = getRemoveRedaction(fileId, annotationId);
handleRemoveFromDictionary(dossierId, fileId, annotationId, removeRedaction.getStatus(),
removeRedaction.isRemoveFromDictionary(), true);
handleRemoveFromDictionary(dossierId, fileId, annotationId, removeRedaction.getStatus(), removeRedaction.isRemoveFromDictionary(), true);
removeRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
if (removeRedaction.getStatus().equals(AnnotationStatus.REQUESTED)) {
@ -443,7 +452,8 @@ public class ManualRedactionService {
@SuppressWarnings("PMD")
public void updateRemoveRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
public void updateRemoveRedactionStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
@ -462,7 +472,6 @@ public class ManualRedactionService {
var redactionLogEntry = redactionLogEntryOptional.get();
if (annotationStatus == AnnotationStatus.APPROVED) {
removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId);
} else if (annotationStatus == AnnotationStatus.DECLINED) {
@ -482,7 +491,9 @@ public class ManualRedactionService {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) {
if (redactionLogEntry.isDossierDictionaryEntry()) {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
} else {
@ -491,7 +502,8 @@ public class ManualRedactionService {
}
public void updateForceRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
public void updateForceRedactionStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
forceRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
@ -501,7 +513,8 @@ public class ManualRedactionService {
}
public void updateLegalBasisChangeStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
public void updateLegalBasisChangeStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
legalBasisChangePersistenceService.updateStatus(fileId, annotationId, annotationStatus);
@ -535,7 +548,8 @@ public class ManualRedactionService {
@SuppressWarnings("PMD")
public void updateAddRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
public void updateAddRedactionStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
@ -562,7 +576,8 @@ public class ManualRedactionService {
}
public void updateResizeRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
public void updateResizeRedactionStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
resizeRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
@ -578,8 +593,9 @@ public class ManualRedactionService {
}
private void handleAddToDictionary(String fileId, String annotationId, String typeId, String value, AnnotationStatus status,
boolean addToDictionary, boolean revert, String dossierId) {
private void handleAddToDictionary(String fileId, String annotationId, String typeId, String value,
AnnotationStatus status, boolean addToDictionary, boolean revert,
String dossierId) {
if (status == AnnotationStatus.APPROVED) {
addRedactionPersistenceService.updateStatus(fileId, annotationId, status, addToDictionary);
@ -596,9 +612,8 @@ public class ManualRedactionService {
}
private void handleRemoveFromDictionary(String dossierId, String fileId, String annotationId, AnnotationStatus status,
boolean removeFromDictionary, boolean revert) {
private void handleRemoveFromDictionary(String dossierId, String fileId, String annotationId,
AnnotationStatus status, boolean removeFromDictionary, boolean revert) {
if (status == AnnotationStatus.APPROVED) {
@ -656,7 +671,6 @@ public class ManualRedactionService {
private CommentEntity addComment(String fileId, String annotationId, String comment, String user) {
var createdComment = commentPersistenceService.insert(CommentEntity.builder()
.text(comment)
.fileId(fileId)
@ -708,4 +722,41 @@ public class ManualRedactionService {
return false;
}
public void updateManualRedactions(String fileId, ManualRedactions manualRedactions) {
manualRedactions.getEntriesToAdd().forEach(e -> {
addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e
.getTextAfter());
});
manualRedactions.getResizeRedactions().forEach(e -> {
resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e
.getTextBefore(), e.getTextAfter());
});
}
private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING);
var analyseRequest = AnalyzeRequest.builder()
.messageType(MessageType.SURROUNDING_TEXT)
.dossierId(dossierId)
.fileId(fileId)
.manualRedactions(manualRedactions)
.build();
try {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> {
message.getMessageProperties().setPriority(1);
return message;
});
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
}

View File

@ -14,6 +14,7 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.MessageType;
public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
@ -47,6 +48,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
.messageType(MessageType.FULL_ANALYSE)
.analysisVersion(100)
.fileId(file.getId())
.dossierId(dossier.getId())
@ -95,6 +97,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
.messageType(MessageType.FULL_ANALYSE)
.analysisVersion(100)
.fileId(file.getId())
.dossierId(dossier.getId())
@ -135,6 +138,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier);
fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
.messageType(MessageType.FULL_ANALYSE)
.analysisVersion(100)
.fileId(file.getId())
.dossierId(dossier.getId())

View File

@ -24,7 +24,7 @@
</modules>
<properties>
<redaction-service.version>3.40.0</redaction-service.version>
<redaction-service.version>3.43.0</redaction-service.version>
<search-service.version>2.10.0</search-service.version>
<pdftron-redaction-service.version>3.17.0</pdftron-redaction-service.version>
<redaction-report-service.version>3.19.0</redaction-report-service.version>