Pull request #251: RED-3210 Bugfix with suggestions

Merge in RED/persistence-service from bugfix/RED-3210 to master

* commit '2dd200bc40c005dd293dd563f9a95d778c1adca4':
  RED-3210 Bugfix with suggestions
This commit is contained in:
Philipp Schramm 2022-02-21 13:58:40 +01:00
commit 06aac4aea3
4 changed files with 135 additions and 180 deletions

View File

@ -104,9 +104,9 @@ public class AddRedactionPersistenceService {
@Transactional
public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) {
public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus, OffsetDateTime processedDate) {
manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus);
manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus, processedDate);
}
@ -121,7 +121,7 @@ public class AddRedactionPersistenceService {
@Transactional
public void approveStatusForRequestedRedactionsWithSameValue(Set<String> fileIds, String value) {
manualRedactionRepository.updateStatus(fileIds, value, AnnotationStatus.REQUESTED, AnnotationStatus.APPROVED);
manualRedactionRepository.updateStatus(fileIds, value, AnnotationStatus.REQUESTED, AnnotationStatus.APPROVED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
}

View File

@ -24,14 +24,13 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
@Modifying
@Query("update ManualRedactionEntryEntity m set m.status = :annotationStatus where m.id = :id")
void updateStatus(AnnotationEntityId id, AnnotationStatus annotationStatus);
@Query("update ManualRedactionEntryEntity m set m.status = :annotationStatus, m.processedDate = :processedDate where m.id = :id")
void updateStatus(AnnotationEntityId id, AnnotationStatus annotationStatus, OffsetDateTime processedDate);
@Modifying
@Query("update ManualRedactionEntryEntity m set m.status = :annotationStatus, m.addToDictionary = :isAddOrRemoveFromDictionary, m.addToDossierDictionary = :isAddOrRemoveFromDossierDictionary where m.id = :id")
void updateStatus(AnnotationEntityId id, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary,
boolean isAddOrRemoveFromDossierDictionary);
void updateStatus(AnnotationEntityId id, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary, boolean isAddOrRemoveFromDossierDictionary);
@Query("select m from ManualRedactionEntryEntity m where m.id = :id and m.softDeletedTime is null")
@ -48,8 +47,7 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
@Modifying
@Query("update ManualRedactionEntryEntity m set m.status = :newStatus where m.id.fileId in :fileIds and m.value = :filterValue and m.addToDictionary = true and m.status = :filterStatus ")
void updateStatus(Set<String> fileIds, String filterValue, AnnotationStatus filterStatus,
AnnotationStatus newStatus);
@Query("update ManualRedactionEntryEntity m set m.status = :newStatus, m.processedDate = :processedDate where m.id.fileId in :fileIds and m.value = :filterValue and m.addToDictionary = true and m.status = :filterStatus ")
void updateStatus(Set<String> fileIds, String filterValue, AnnotationStatus filterStatus, AnnotationStatus newStatus, OffsetDateTime processedDate);
}

View File

@ -38,8 +38,7 @@ public class AnalysisFlagsCalculationService {
var viewedPagesForCurrentAssignee = viewedPagesPersistenceService.findViewedPages(fileId, file.getAssignee());
Map<Integer, OffsetDateTime> viewedPages = viewedPagesForCurrentAssignee.stream()
.collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime));
Map<Integer, OffsetDateTime> viewedPages = viewedPagesForCurrentAssignee.stream().collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime));
boolean hasRedactions = false;
boolean hasHints = false;
@ -57,11 +56,9 @@ public class AnalysisFlagsCalculationService {
String type = getType(entry.getType());
var lastChange = entry.getChanges().isEmpty() ? null : entry.getChanges()
.get(entry.getChanges().size() - 1);
var lastChange = entry.getChanges().isEmpty() ? null : entry.getChanges().get(entry.getChanges().size() - 1);
if (lastChange != null && (lastModification == null || lastChange.getDateTime()
.isAfter(lastModification))) {
if (lastChange != null && (lastModification == null || lastChange.getDateTime().isAfter(lastModification))) {
lastModification = lastChange.getDateTime();
}
@ -77,7 +74,7 @@ public class AnalysisFlagsCalculationService {
hasImages = true;
}
if (!hasSuggestions && lastChange != null && lastChange.getType() != ChangeType.REMOVED && entry.getManualChanges()
if (!hasSuggestions && (lastChange == null || lastChange.getType() != ChangeType.REMOVED) && entry.getManualChanges()
.stream()
.anyMatch(e -> e.getAnnotationStatus() == AnnotationStatus.REQUESTED)) {
hasSuggestions = true;
@ -87,9 +84,7 @@ public class AnalysisFlagsCalculationService {
hasComments = true;
}
var viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions()
.get(0)
.getPage());
var viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions().get(0).getPage());
if (file.getWorkflowStatus() != WorkflowStatus.APPROVED && lastChange != null && lastChange.getDateTime() != null && viewedPage != null && viewedPage.isBefore(lastChange.getDateTime())) {
hasUpdates = true;
@ -98,7 +93,6 @@ public class AnalysisFlagsCalculationService {
}
log.info("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime);
if (file.isHasRedactions() == hasRedactions && file.isHasHints() == hasHints && file.isHasImages() == hasImages && file.isHasSuggestions() == hasSuggestions && file.isHasAnnotationComments() == hasComments && file.isHasUpdates() == hasUpdates) {
log.info("Nothing Changed for file: {}", fileId);
} else {
@ -110,8 +104,7 @@ public class AnalysisFlagsCalculationService {
lastModification = lastManualRedactionTime;
}
if (lastModification != null && (file.getAnnotationModificationDate() == null || file.getAnnotationModificationDate()
.isBefore(lastModification))) {
if (lastModification != null && (file.getAnnotationModificationDate() == null || file.getAnnotationModificationDate().isBefore(lastModification))) {
fileStatusPersistenceService.setLastAnnotationModificationDateForFile(fileId, lastModification);
}
}

View File

@ -144,8 +144,7 @@ public class ManualRedactionService {
}
public ManualAddResponse addRemoveRedaction(String dossierId, String fileId,
RemoveRedactionRequest removeRedactionRequest) {
public ManualAddResponse addRemoveRedaction(String dossierId, String fileId, RemoveRedactionRequest removeRedactionRequest) {
dossierPersistenceService.getAndValidateDossier(dossierId);
@ -174,15 +173,11 @@ public class ManualRedactionService {
reprocess(dossierId, fileId);
}
return ManualAddResponse.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.commentId(commentId)
.build();
return ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build();
}
public ManualAddResponse addForceRedaction(String dossierId, String fileId,
ForceRedactionRequest forceRedactionRequest) {
public ManualAddResponse addForceRedaction(String dossierId, String fileId, ForceRedactionRequest forceRedactionRequest) {
dossierPersistenceService.getAndValidateDossier(dossierId);
@ -207,15 +202,11 @@ public class ManualRedactionService {
reprocess(dossierId, fileId);
}
return ManualAddResponse.builder()
.annotationId(forceRedactionRequest.getAnnotationId())
.commentId(commentId)
.build();
return ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build();
}
public ManualAddResponse addLegalBasisChange(String dossierId, String fileId,
LegalBasisChangeRequest legalBasisChangeRequest) {
public ManualAddResponse addLegalBasisChange(String dossierId, String fileId, LegalBasisChangeRequest legalBasisChangeRequest) {
dossierPersistenceService.getAndValidateDossier(dossierId);
@ -236,15 +227,11 @@ public class ManualRedactionService {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return ManualAddResponse.builder()
.annotationId(legalBasisChangeRequest.getAnnotationId())
.commentId(commentId)
.build();
return ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build();
}
public ManualAddResponse addImageRecategorization(String dossierId, String fileId,
ImageRecategorizationRequest imageRecategorizationRequest) {
public ManualAddResponse addImageRecategorization(String dossierId, String fileId, ImageRecategorizationRequest imageRecategorizationRequest) {
dossierPersistenceService.getAndValidateDossier(dossierId);
@ -266,43 +253,7 @@ public class ManualRedactionService {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return ManualAddResponse.builder()
.annotationId(imageRecategorizationRequest.getAnnotationId())
.commentId(commentId)
.build();
}
public ManualAddResponse addResizeRedaction(String dossierId, String fileId,
ResizeRedactionRequest resizeRedactionRequest) {
dossierPersistenceService.getAndValidateDossier(dossierId);
OffsetDateTime now = OffsetDateTime.now();
resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
Long commentId = null;
if (resizeRedactionRequest.getComment() != null) {
commentId = addComment(fileId, resizeRedactionRequest.getAnnotationId(), resizeRedactionRequest.getComment(), resizeRedactionRequest.getUser()).getId();
}
if (resizeRedactionRequest.getStatus().equals(AnnotationStatus.REQUESTED)) {
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, now, true);
} else {
fileStatusPersistenceService.updateLastManualRedaction(fileId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
var loaded = convert(getResizeRedaction(fileId, resizeRedactionRequest.getAnnotationId()), ManualResizeRedaction.class, new ManualResizeRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of(loaded)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
return ManualAddResponse.builder()
.annotationId(resizeRedactionRequest.getAnnotationId())
.commentId(commentId)
.build();
return ManualAddResponse.builder().annotationId(imageRecategorizationRequest.getAnnotationId()).commentId(commentId).build();
}
@ -474,9 +425,37 @@ public class ManualRedactionService {
}
public ManualAddResponse addResizeRedaction(String dossierId, String fileId, ResizeRedactionRequest resizeRedactionRequest) {
dossierPersistenceService.getAndValidateDossier(dossierId);
OffsetDateTime now = OffsetDateTime.now();
resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
Long commentId = null;
if (resizeRedactionRequest.getComment() != null) {
commentId = addComment(fileId, resizeRedactionRequest.getAnnotationId(), resizeRedactionRequest.getComment(), resizeRedactionRequest.getUser()).getId();
}
if (resizeRedactionRequest.getStatus().equals(AnnotationStatus.REQUESTED)) {
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, now, true);
} else {
fileStatusPersistenceService.updateLastManualRedaction(fileId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
var loaded = convert(getResizeRedaction(fileId, resizeRedactionRequest.getAnnotationId()), ManualResizeRedaction.class, new ManualResizeRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of(loaded)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
return ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build();
}
@SuppressWarnings("PMD")
public void updateRemoveRedactionStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
public void updateRemoveRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
@ -517,8 +496,7 @@ public class ManualRedactionService {
}
public void updateForceRedactionStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
public void updateForceRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
forceRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
@ -528,9 +506,14 @@ public class ManualRedactionService {
}
public ManualRedactions getManualRedactions(String fileId) {
return manualRedactionProviderService.getManualRedactions(fileId);
}
@Transactional
public void updateLegalBasisChangeStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
public void updateLegalBasisChangeStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
legalBasisChangePersistenceService.updateStatus(fileId, annotationId, annotationStatus);
@ -540,8 +523,7 @@ public class ManualRedactionService {
}
public void updateImageRecategorizationStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
public void updateImageRecategorizationStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
ManualImageRecategorizationEntity imageRecategorization = recategorizationPersistenceService.findRecategorization(fileId, annotationId);
@ -563,8 +545,18 @@ public class ManualRedactionService {
}
public void updateResizeRedactionStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
@Transactional
public void updateSurroundingText(String fileId, ManualRedactions manualRedactions) {
// These are marked as processed once surrounding text is computed ( TBD if this is correct ? )
manualRedactions.getEntriesToAdd()
.forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
manualRedactions.getResizeRedactions()
.forEach(e -> resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
}
public void updateResizeRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
resizeRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
@ -574,15 +566,8 @@ public class ManualRedactionService {
}
public ManualRedactions getManualRedactions(String fileId) {
return manualRedactionProviderService.getManualRedactions(fileId);
}
@SuppressWarnings("PMD")
public void updateAddRedactionStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
public void updateAddRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
@ -601,10 +586,10 @@ public class ManualRedactionService {
}
}
addRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
addRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
boolean hasSuggestions = calculateHasSuggestions(fileId);
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasSuggestions);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -614,29 +599,60 @@ public class ManualRedactionService {
public void updateProcessedDate(String fileId, ManualRedactions manualRedactions) {
// These are marked as processed once analysis completes, not when they are set as approved
if(manualRedactions != null) {
if (manualRedactions != null) {
if(manualRedactions.getIdsToRemove() != null) {
if (manualRedactions.getIdsToRemove() != null) {
manualRedactions.getIdsToRemove().forEach(removeRedactionPersistenceService::markAsProcessed);
}
if(manualRedactions.getForceRedactions() != null) {
if (manualRedactions.getForceRedactions() != null) {
manualRedactions.getForceRedactions().forEach(forceRedactionPersistenceService::markAsProcessed);
}
if(manualRedactions.getImageRecategorization() != null) {
if (manualRedactions.getImageRecategorization() != null) {
manualRedactions.getImageRecategorization().forEach(recategorizationPersistenceService::markAsProcessed);
}
}
}
@Transactional
public void updateSurroundingText(String fileId, ManualRedactions manualRedactions) {
private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) {
// These are marked as processed once surrounding text is computed ( TBD if this is correct ? )
manualRedactions.getEntriesToAdd()
.forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
manualRedactions.getResizeRedactions()
.forEach(e -> resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING);
var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId).manualRedactions(manualRedactions).build();
try {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> {
message.getMessageProperties().setPriority(1);
return message;
});
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
private void approveStatusForRedactionsWithSameValue(DossierEntity dossier, boolean addToDictionary, boolean addToDossierDictionary, String value) {
List<DossierEntity> dossiers = new ArrayList<>();
if (addToDictionary) {
dossiers = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId()).getDossiers();
}
if (addToDossierDictionary) {
dossiers.add(dossier);
}
Set<String> fileIds = new HashSet<>();
for (DossierEntity d : dossiers) {
var files = fileStatusService.getDossierStatus(d.getId());
files.forEach(f -> fileIds.add(f.getId()));
}
if (!fileIds.isEmpty()) {
log.debug("Approve status for requested redactions with same value '{}' for files {}", value, fileIds);
addRedactionPersistenceService.approveStatusForRequestedRedactionsWithSameValue(fileIds, value);
}
}
@ -646,37 +662,7 @@ public class ManualRedactionService {
}
private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) {
if (redactionLogEntry.isDossierDictionaryEntry()) {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
} else {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
}
}
private void handleAddToDictionary(String fileId, String annotationId, String typeId, String value,
AnnotationStatus status, boolean addToDictionary, boolean addToDossierDictionary,
boolean revert, String dossierId) {
if (status == AnnotationStatus.APPROVED) {
addRedactionPersistenceService.updateStatus(fileId, annotationId, status, addToDictionary, addToDossierDictionary);
if (addToDictionary || addToDossierDictionary) {
if (revert) {
removeFromDictionary(typeId, value, dossierId, fileId);
} else {
addToDictionary(typeId, value, dossierId, fileId);
}
}
}
}
private void handleRemoveFromDictionary(String dossierId, String fileId, String annotationId,
AnnotationStatus status, boolean removeFromDictionary, boolean revert) {
private void handleRemoveFromDictionary(String dossierId, String fileId, String annotationId, AnnotationStatus status, boolean removeFromDictionary, boolean revert) {
if (status == AnnotationStatus.APPROVED) {
@ -781,56 +767,34 @@ public class ManualRedactionService {
return true;
}
}
return false;
}
private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) {
private void handleAddToDictionary(String fileId, String annotationId, String typeId, String value, AnnotationStatus status, boolean addToDictionary,
boolean addToDossierDictionary, boolean revert, String dossierId) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING);
if (status == AnnotationStatus.APPROVED) {
addRedactionPersistenceService.updateStatus(fileId, annotationId, status, addToDictionary, addToDossierDictionary);
var analyseRequest = AnalyzeRequest.builder()
.messageType(MessageType.SURROUNDING_TEXT)
.dossierId(dossierId)
.fileId(fileId)
.manualRedactions(manualRedactions)
.build();
if (addToDictionary || addToDossierDictionary) {
if (revert) {
removeFromDictionary(typeId, value, dossierId, fileId);
} else {
addToDictionary(typeId, value, dossierId, fileId);
}
}
try {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> {
message.getMessageProperties().setPriority(1);
return message;
});
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
private void approveStatusForRedactionsWithSameValue(DossierEntity dossier, boolean addToDictionary,
boolean addToDossierDictionary, String value) {
private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) {
List<DossierEntity> dossiers = new ArrayList<>();
if (addToDictionary) {
dossiers = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId())
.getDossiers();
}
if (addToDossierDictionary) {
dossiers.add(dossier);
}
Set<String> fileIds = new HashSet<>();
for (DossierEntity d : dossiers) {
var files = fileStatusService.getDossierStatus(d.getId());
files.forEach(f -> fileIds.add(f.getId()));
}
if (!fileIds.isEmpty()) {
log.debug("Approve status for requested redactions with same value '{}' for files {}", value, fileIds);
addRedactionPersistenceService.approveStatusForRequestedRedactionsWithSameValue(fileIds, value);
if (redactionLogEntry.isDossierDictionaryEntry()) {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
} else {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
}
}