Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-7317: Endpoint to change entity types of dict-based annotations

Browse Source 
* added undo functionality
* fixed tests
* added recategorize with dictionary tests
This commit is contained in:
Kilian Schuettler 2023-08-30 13:02:40 +02:00
parent aab2971d6e
commit ea5f79d67c
15 changed files with 863 additions and 702 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java vendored
View File

@ -20,6 +20,7 @@ import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionUndoService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
@ -44,22 +45,26 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Remo
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ResizeRedactionRequestModel;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RestController
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualRedactionController implements ManualRedactionResource {
private static final String FILE_ID = "fileId";
private static final String DOSSIER_ID = "dossierId";
private static final String ANNOTATION_ID = "annotationId";
private final ManualRedactionService manualRedactionService;
private final ManualRedactionUndoService manualRedactionUndoService;
private final DossierManagementService dossierManagementService;
private final AuditPersistenceService auditPersistenceService;
private final AccessControlService accessControlService;
final static String FILE_ID = "fileId";
final static String DOSSIER_ID = "dossierId";
final static String ANNOTATION_ID = "annotationId";
ManualRedactionService manualRedactionService;
ManualRedactionUndoService manualRedactionUndoService;
DossierManagementService dossierManagementService;
AuditPersistenceService auditPersistenceService;
AccessControlService accessControlService;
CommentService commentService;
@Override
@ -90,7 +95,7 @@ public class ManualRedactionController implements ManualRedactionResource {
.message("Comment was removed.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build());
manualRedactionService.deleteComment(dossierId, fileId, List.of(Long.valueOf(commentId)));
commentService.deleteComment(dossierId, fileId, List.of(Long.valueOf(commentId)));
}
@ -107,7 +112,7 @@ public class ManualRedactionController implements ManualRedactionResource {
@PreAuthorize("hasAuthority('" + READ_MANUAL_REDACTIONS + "')")
public Comments getComments(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
return manualRedactionService.getComments(fileId);
return commentService.getComments(fileId);
}
@ -121,7 +126,7 @@ public class ManualRedactionController implements ManualRedactionResource {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewerOrApprover(dossierId, fileId);
var response = manualRedactionService.addComment(dossierId,
var response = commentService.addComment(dossierId,
fileId,
annotationId,
CommentRequest.builder().user(KeycloakSecurity.getUserId()).text(addCommentRequest.getText()).build());

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java
View File

@ -9,7 +9,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;

128
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CommentService.java Normal file
View File

@ -0,0 +1,128 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.CommentRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comments;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class CommentService {
CommentPersistenceService commentPersistenceService;
FileStatusPersistenceService fileStatusPersistenceService;
FileStatusService fileStatusService;
RedactionLogService redactionLogService;
FileManagementStorageService fileManagementStorageService;
@Transactional
public void deleteComment(String dossierId, String fileId, List<Long> commentIds) {
for (var commentId : commentIds) {
commentPersistenceService.softDelete(commentId, OffsetDateTime.now());
}
// update indicator
fileStatusPersistenceService.updateHasComments(fileId, commentPersistenceService.fileHasComments(fileId));
// This is an ugly workaround, don't even look at it!
// Basically we should change how comments work and not merge them into the redaction log.
// if file is currently not analyzing, we can quickly change the redaction log, else we must wait for the analysis to finish.
if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) {
reprocess(dossierId, fileId);
return;
}
fileStatusService.setStatusProcessing(fileId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
redactionLog.getRedactionLogEntry().forEach(entry -> entry.getComments().removeIf(comment -> commentIds.contains(comment.getId())));
fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog);
fileStatusService.setStatusProcessed(fileId);
}
@Transactional
public Comments getComments(String fileId) {
return new Comments(commentPersistenceService.findCommentsByFileID(fileId, false)
.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> MagicConverter.convert(entry.getValue(), Comment.class))));
}
@Transactional
public CommentEntity addComment(String dossierId, String fileId, String annotationId, CommentRequest commentRequest) {
CommentEntity createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser());
fileStatusPersistenceService.updateHasComments(fileId, true);
// This is an ugly workaround, don't even look at it!
// Basically we should change how comments work and not merge them into the redaction log.
if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) {
reprocess(dossierId, fileId);
return createdComment;
}
fileStatusService.setStatusProcessing(fileId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.forEach(entry -> entry.getComments().add(MagicConverter.convert(createdComment, RedactionLogComment.class)));
fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog);
fileStatusService.setStatusProcessed(fileId);
return createdComment;
}
public Long addCommentAndGetId(String fileId, String annotationId, String comment, String user) {
if (comment == null) {
return null;
}
return addComment(fileId, annotationId, comment, user).getId();
}
private CommentEntity addComment(String fileId, String annotationId, String comment, String user) {
if (comment == null) {
return null;
}
return commentPersistenceService.insert(CommentEntity.builder()
.text(comment)
.fileId(fileId)
.annotationId(annotationId)
.user(user)
.date(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))
.build());
}
private void reprocess(String dossierId, String fileId) {
fileStatusService.setStatusReprocessForManual(dossierId, fileId, true);
}
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@ -13,7 +12,6 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
@ -25,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.NerServic
import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse;
import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest;
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;

250
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionDictionaryUpdateHandler.java
View File

@ -2,18 +2,26 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.StopwordService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithAddToDictionary;
@ -23,71 +31,231 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import feign.FeignException;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualRedactionDictionaryUpdateHandler {
private final DictionaryManagementService dictionaryManagementService;
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final FileStatusPersistenceService fileStatusPersistenceService;
private final ResizeRedactionPersistenceService resizeRedactionPersistenceService;
private final DossierPersistenceService dossierPersistenceService;
DictionaryManagementService dictionaryManagementService;
DictionaryPersistenceService dictionaryPersistenceService;
FileStatusPersistenceService fileStatusPersistenceService;
ResizeRedactionPersistenceService resizeRedactionPersistenceService;
DossierPersistenceService dossierPersistenceService;
StopwordService stopwordService;
public Set<String> handleAddToDictionaryAndReturnModifiedTypeIds(String fileId, String annotationId, ManualRequestWithAddToDictionary manualRequestWithAddToDictionary) {
if (!manualRequestWithAddToDictionary.isApproved()) {
return null;
}
public Set<String> handleAddToDictionaryAndReturnModifiedTypeIds(String fileId, String value, ManualRequestWithAddToDictionary manualRequestWithAddToDictionary) {
if (!manualRequestWithAddToDictionary.isAddToDictionary()) {
addRedactionPersistenceService.updateStatus(fileId, annotationId, manualRequestWithAddToDictionary.getStatus(), false, null);
return null;
return Collections.emptySet();
}
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
if (manualRequestWithAddToDictionary.isAddToAllDossiers()) {
var dictionaryEntriesToUnDelete = dictionaryManagementService.getAllEntriesInDossierTemplate(manualRequestWithAddToDictionary.getDossierTemplateTypeId(),
manualRequestWithAddToDictionary.getValue());
List<DictionaryEntry> dictionaryEntriesToUnDelete = dictionaryManagementService.getAllEntriesInDossierTemplate(manualRequestWithAddToDictionary.getDossierTemplateTypeId(),
value);
dictionaryEntriesToUnDelete.forEach(entry -> {
typeIdsOfModifiedDictionaries.add(entry.getTypeId());
addToDictionary(entry.getTypeId(), manualRequestWithAddToDictionary.getValue(), manualRequestWithAddToDictionary.getDossierId(), fileId, manualRequestWithAddToDictionary.getDictionaryEntryType());
addToDictionary(entry.getTypeId(),
value,
manualRequestWithAddToDictionary.getDossierId(),
fileId,
manualRequestWithAddToDictionary.getDictionaryEntryType());
});
addToDictionary(manualRequestWithAddToDictionary.getDossierTemplateTypeId(),
manualRequestWithAddToDictionary.getValue(),
value,
manualRequestWithAddToDictionary.getDossierId(),
fileId,
manualRequestWithAddToDictionary.getDictionaryEntryType());
typeIdsOfModifiedDictionaries.add(manualRequestWithAddToDictionary.getDossierTemplateTypeId());
return typeIdsOfModifiedDictionaries;
}
addToDictionary(manualRequestWithAddToDictionary.getDossierTemplateTypeId() + ":" + manualRequestWithAddToDictionary.getDossierId(),
manualRequestWithAddToDictionary.getValue(),
value,
manualRequestWithAddToDictionary.getDossierId(),
fileId,
manualRequestWithAddToDictionary.getDictionaryEntryType());
typeIdsOfModifiedDictionaries.add(manualRequestWithAddToDictionary.getDossierTemplateTypeId() + ":" + manualRequestWithAddToDictionary.getDossierId());
return typeIdsOfModifiedDictionaries;
}
public Set<String> handleRemoveFromDictionaryAndReturnModifiedTypeIds(RedactionLogEntry redactionLogEntry,
String fileId,
String dossierId,
String dossierTemplateId,
ManualRequestWithRemoveFromDictionary manualRequestWithRemoveFromDictionary) {
if (!manualRequestWithRemoveFromDictionary.isRemoveFromDictionary()) {
return Collections.emptySet();
}
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
if (manualRequestWithRemoveFromDictionary.isRemoveFromAllDossiers()) {
var dictionaryEntriesToRemove = dictionaryManagementService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(), dossierTemplateId),
redactionLogEntry.getValue());
dictionaryEntriesToRemove.forEach(entry -> {
typeIdsOfModifiedDictionaries.add(entry.getTypeId());
removeFromDictionary(entry.getTypeId(), entry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
});
} else {
typeIdsOfModifiedDictionaries.add(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId));
removeFromDictionary(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
}
// This is needed to remove resizeRedactions with addToDictionary.
removeResizeRedactionsWithAddToDictionary(dossierTemplateId, redactionLogEntry.getValue());
return typeIdsOfModifiedDictionaries;
}
public Set<String> updateDictionaryForResizeRedactions(String dossierId, String fileId, ManualResizeRedactionEntity resizeRedaction, RedactionLogEntry redactionLogEntry) {
if (resizeRedaction.getUpdateDictionary() != null && resizeRedaction.getUpdateDictionary() && resizeRedaction.getStatus()
.equals(AnnotationStatus.APPROVED) && (redactionLogEntry.isDictionaryEntry() || redactionLogEntry.isDossierDictionaryEntry())) {
var dossier = dossierPersistenceService.findByDossierId(dossierId);
var typeId = buildTypeId(redactionLogEntry, resizeRedaction, dossier);
var newValue = resizeRedaction.getValue();
var oldValue = redactionLogEntry.getValue();
var dictionaryEntryType = getDictionaryEntryType(redactionLogEntry);
boolean isShrinking = oldValue != null && oldValue.length() > newValue.length();
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
if (isShrinking) {
log.info("Remove old value '{}' from dictionary", oldValue);
removeFromDictionary(typeId, oldValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeId);
if (resizeRedaction.isAddToAllDossiers() && redactionLogEntry.isDictionaryEntry()) {
String dossierTemplateId = dossier.getDossierTemplateId();
var dossiersOfThisDossierTemplate = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
var type = redactionLogEntry.getType();
dossiersOfThisDossierTemplate.forEach(dossierEntity -> {
var typeIdOfDossierEntity = toTypeId(type, dossierTemplateId, dossierEntity.getId());
removeFromDictionary(typeIdOfDossierEntity, oldValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeIdOfDossierEntity);
});
}
}
log.info("Add new value '{}' to dictionary", newValue);
addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeId);
return typeIdsOfModifiedDictionaries;
}
return Collections.emptySet();
}
private String buildTypeId(RedactionLogEntry redactionLogEntry, ManualResizeRedactionEntity resizeRedaction, DossierEntity dossier) {
if (resizeRedaction.isAddToAllDossiers()) {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
} else {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
}
}
private DictionaryEntryType getDictionaryEntryType(RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.isRecommendation() && redactionLogEntry.isFalsePositive()) {
return DictionaryEntryType.FALSE_RECOMMENDATION;
} else if (redactionLogEntry.isFalsePositive()) {
return DictionaryEntryType.FALSE_POSITIVE;
} else {
return DictionaryEntryType.ENTRY;
}
}
public boolean revertAddToDictionary(String fileId, String dossierId, ManualRedactionEntryEntity manualRedactionToRevert) {
if (!manualRedactionToRevert.getStatus().equals(AnnotationStatus.APPROVED) || manualRedactionToRevert.isAddToDictionary()) {
if (!manualRedactionToRevert.isAddToDictionary()) {
return false;
}
manualRedactionToRevert.getTypeIdsOfModifiedDictionaries().forEach(typeId -> {
removeFromDictionary(typeId, manualRedactionToRevert.getValue(), dossierId, fileId, manualRedactionToRevert.getDictionaryEntryType());
});
revertAddToDictionary(manualRedactionToRevert.getValue(),
manualRedactionToRevert.getDictionaryEntryType(),
fileId,
dossierId,
manualRedactionToRevert.getTypeIdsOfModifiedDictionaries());
return true;
}
public void revertAddToDictionary(String value, DictionaryEntryType dictionaryEntryType, String fileId, String dossierId, Set<String> typeIdsOfModifiedDictionaries) {
typeIdsOfModifiedDictionaries.forEach(typeId -> {
removeFromDictionary(typeId, value, dossierId, fileId, dictionaryEntryType);
});
}
public boolean revertRemoveFromDictionary(String value, String dossierId, String fileId, IdRemovalEntity idRemoval) {
if (!idRemoval.isRemoveFromDictionary()) {
return false;
}
revertRemoveFromDictionary(value, dossierId, fileId, idRemoval.getTypeIdsOfModifiedDictionaries());
return true;
}
public void revertRemoveFromDictionary(String value, String dossierId, String fileId, Set<String> typeIdsOfModifiedDictionaries) {
typeIdsOfModifiedDictionaries.forEach(changedTypeId -> addToDictionary(changedTypeId, value, dossierId, fileId, DictionaryEntryType.ENTRY));
}
public void validateDictionariesForDelete(ManualRequestWithRemoveFromDictionary request, RedactionLogEntry redactionLogEntry, String dossierTemplateId) {
if (request.isRemoveFromDictionary()) {
var dossierTemplateTypeId = toTypeId(redactionLogEntry.getType(), dossierTemplateId);
dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(dossierTemplateTypeId, request.isRemoveFromDictionary(), request.isRemoveFromAllDossiers());
}
}
public void validateDictionariesForAdd(ManualRequestWithAddToDictionary addRedactionRequest, String value) {
if (addRedactionRequest.isAddToDictionary()) {
if (addRedactionRequest.isAddToAllDossiers()) {
// validate add to dossier template dictionaries
dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(addRedactionRequest.getDossierTemplateTypeId());
}
validateDictionary(addRedactionRequest, value);
}
}
private void validateDictionary(ManualRequestWithAddToDictionary addRedactionRequest, String value) {
try {
if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(value)) {
throw new ConflictException("The entry you are trying to add is a stopword");
}
dictionaryPersistenceService.getType(addRedactionRequest.getDossierTemplateTypeId());
} catch (NotFoundException e) {
throw new BadRequestException("Invalid type: " + addRedactionRequest.getDossierTemplateTypeId());
}
}
private void addToDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) {
try {
@ -110,41 +278,6 @@ public class ManualRedactionDictionaryUpdateHandler {
}
public Set<String> handleRemoveFromDictionaryAndReturnModifiedTypeIds(RedactionLogEntry redactionLogEntry,
String fileId,
String dossierId,
String dossierTemplateId,
ManualRequestWithRemoveFromDictionary manualRequestWithRemoveFromDictionary) {
if (!manualRequestWithRemoveFromDictionary.isApproved()) {
return null;
}
if (!manualRequestWithRemoveFromDictionary.isRemoveFromDictionary()) {
return null;
}
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
if (manualRequestWithRemoveFromDictionary.isRemoveFromAllDossiers()) {
var dictionaryEntriesToRemove = dictionaryManagementService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(), dossierTemplateId),
redactionLogEntry.getValue());
dictionaryEntriesToRemove.forEach(entry -> {
typeIdsOfModifiedDictionaries.add(entry.getTypeId());
removeFromDictionary(entry.getTypeId(), entry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
});
} else {
typeIdsOfModifiedDictionaries.add(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId));
removeFromDictionary(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
}
// This is needed to remove resizeRedactions with addToDictionary.
removeResizeRedactionsWithAddToDictionary(dossierTemplateId, redactionLogEntry.getValue());
return typeIdsOfModifiedDictionaries;
}
private void removeResizeRedactionsWithAddToDictionary(String dossierTemplateId, String redactionLogEntryValue) {
var resizeRedactionsWithSameValue = resizeRedactionPersistenceService.findByAnnotationStatusAndValue(AnnotationStatus.APPROVED, redactionLogEntryValue);
@ -157,5 +290,4 @@ public class ManualRedactionDictionaryUpdateHandler {
});
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionProviderService.java → persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
package com.iqser.red.service.persistence.management.v1.processor.service.manualredactions;
import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;

767
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java
View File

@ -1,44 +1,24 @@
package com.iqser.red.service.persistence.management.v1.processor.service.manualredactions;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.StopwordService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
@ -47,53 +27,44 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.BaseManualRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.CommentRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comments;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithAddToDictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import feign.FeignException;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualRedactionService {
private final DossierPersistenceService dossierPersistenceService;
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final RemoveRedactionPersistenceService removeRedactionPersistenceService;
private final ForceRedactionPersistenceService forceRedactionPersistenceService;
private final CommentPersistenceService commentPersistenceService;
private final FileStatusPersistenceService fileStatusPersistenceService;
private final DictionaryPersistenceService dictionaryPersistenceService;
private final FileManagementStorageService fileManagementStorageService;
private final ImageRecategorizationPersistenceService recategorizationPersistenceService;
private final LegalBasisChangePersistenceService legalBasisChangePersistenceService;
private final ResizeRedactionPersistenceService resizeRedactionPersistenceService;
private final FileStatusService fileStatusService;
private final ManualRedactionProviderService manualRedactionProviderService;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
private final StopwordService stopwordService;
private final RedactionLogService redactionLogService;
private final DictionaryManagementService dictionaryManagementService;
private final HashFunction hashFunction = Hashing.murmur3_128();
private final ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler;
DossierPersistenceService dossierPersistenceService;
AddRedactionPersistenceService addRedactionPersistenceService;
RemoveRedactionPersistenceService removeRedactionPersistenceService;
ForceRedactionPersistenceService forceRedactionPersistenceService;
CommentService commentService;
ImageRecategorizationPersistenceService recategorizationPersistenceService;
LegalBasisChangePersistenceService legalBasisChangePersistenceService;
ResizeRedactionPersistenceService resizeRedactionPersistenceService;
FileStatusService fileStatusService;
ManualRedactionProviderService manualRedactionProviderService;
AnalysisFlagsCalculationService analysisFlagsCalculationService;
RedactionLogService redactionLogService;
DictionaryManagementService dictionaryManagementService;
HashFunction hashFunction = Hashing.murmur3_128();
ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler;
@Transactional
@ -104,24 +75,22 @@ public class ManualRedactionService {
dossierPersistenceService.getAndValidateDossier(dossierId);
for (AddRedactionRequest addRedactionRequest : addRedactionRequests) {
validateDictionaries(addRedactionRequest);
manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue());
validatePositions(fileId, addRedactionRequest);
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
annotationId,
addRedactionRequest);
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId, addRedactionRequest.getValue(), addRedactionRequest);
addRedactionPersistenceService.updateStatus(fileId,
annotationId,
addRedactionRequest.getStatus(),
typeIdsOfModifiedDictionaries != null,
!typeIdsOfModifiedDictionaries.isEmpty(),
typeIdsOfModifiedDictionaries);
Long commentId = addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser());
Long commentId = commentService.addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
}
@ -136,28 +105,202 @@ public class ManualRedactionService {
}
private void validateDictionaries(ManualRequestWithAddToDictionary addRedactionRequest) {
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId, List<RemoveRedactionRequest> removeRedactionRequests) {
if (addRedactionRequest.isAddToDictionary()) {
if (addRedactionRequest.isAddToAllDossiers()) {
// validate add to dossier template dictionaries
dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(addRedactionRequest.getDossierTemplateTypeId());
var response = new ArrayList<ManualAddResponse>();
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossier.getId(), fileId);
var requiresReAnalysis = false;
var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId);
//validate removing from dossier template dictionary
for (RemoveRedactionRequest removeRedactionRequest : removeRedactionRequests) {
RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId());
manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest, redactionLogEntry, dossier.getDossierTemplateId());
removeRedactionPersistenceService.insert(fileId, removeRedactionRequest);
if (manualAddRedactionsContains(manualRedactions, removeRedactionRequest.getAnnotationId())) {
log.info("hard delete ManualRedactions for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
manualRedactionProviderService.hardDeleteManualRedactions(fileId, removeRedactionRequest.getAnnotationId());
requiresReAnalysis = true;
continue;
}
validateDictionary(addRedactionRequest);
log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
Long commentId = commentService.addCommentAndGetId(fileId,
removeRedactionRequest.getAnnotationId(),
removeRedactionRequest.getComment(),
removeRedactionRequest.getUser());
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(//
redactionLogEntry,//
fileId,//
dossierId, //
dossier.getDossierTemplateId(),//
removeRedactionRequest);
boolean removedFromDictionary = !typeIdsOfModifiedDictionaries.isEmpty();
removeRedactionPersistenceService.updateStatus(fileId,
removeRedactionRequest.getAnnotationId(),
removeRedactionRequest.getStatus(),
removedFromDictionary,
typeIdsOfModifiedDictionaries);
requiresReAnalysis = requiresReAnalysis || removedFromDictionary;
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
if (requiresReAnalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
private void validateDictionary(ManualRequestWithAddToDictionary addRedactionRequest) {
@Transactional
public List<ManualAddResponse> addForceRedaction(String dossierId, String fileId, List<ForceRedactionRequest> forceRedactionRequests) {
try {
if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) {
throw new ConflictException("The entry you are trying to add is a stopword");
}
dictionaryPersistenceService.getType(addRedactionRequest.getDossierTemplateTypeId());
} catch (NotFoundException e) {
throw new BadRequestException("Invalid type: " + addRedactionRequest.getDossierTemplateTypeId());
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var requiresReanalysis = false;
for (var forceRedactionRequest : forceRedactionRequests) {
forceRedactionPersistenceService.insert(fileId, forceRedactionRequest);
Long commentId = commentService.addCommentAndGetId(fileId,
forceRedactionRequest.getAnnotationId(),
forceRedactionRequest.getComment(),
forceRedactionRequest.getUser());
requiresReanalysis = requiresReanalysis || forceRedactionRequest.isApproved();
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
if (requiresReanalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
@Transactional
public List<ManualAddResponse> addLegalBasisChange(String dossierId, String fileId, List<LegalBasisChangeRequest> legalBasisChangeRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
for (var legalBasisChangeRequest : legalBasisChangeRequests) {
legalBasisChangePersistenceService.insert(fileId, legalBasisChangeRequest);
Long commentId = commentService.addCommentAndGetId(fileId,
legalBasisChangeRequest.getAnnotationId(),
legalBasisChangeRequest.getComment(),
legalBasisChangeRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
@Transactional
public List<ManualAddResponse> addRecategorization(String dossierId, String fileId, List<RecategorizationRequest> recategorizationRequests) {
var response = new ArrayList<ManualAddResponse>();
var requiresReanalysis = false;
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
for (var recategorizationRequest : recategorizationRequests) {
RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, recategorizationRequest.getAnnotationId());
manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(recategorizationRequest, redactionLogEntry.getValue());
manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(recategorizationRequest, redactionLogEntry, dossier.getDossierTemplateId());
recategorizationPersistenceService.insert(fileId, recategorizationRequest);
Set<String> typeIdsOfDictionariesWithAdd = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId, redactionLogEntry.getValue(), recategorizationRequest);
Set<String> typeIdsOfDictionariesWithDelete = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(redactionLogEntry,
fileId,
recategorizationRequest.getDossierId(),
dossier.getDossierTemplateId(),
recategorizationRequest);
recategorizationPersistenceService.updateStatus(fileId,
recategorizationRequest.getAnnotationId(),
AnnotationStatus.APPROVED,
typeIdsOfDictionariesWithAdd != null,
typeIdsOfDictionariesWithAdd,
typeIdsOfDictionariesWithDelete);
Long commentId = commentService.addCommentAndGetId(fileId,
recategorizationRequest.getAnnotationId(),
recategorizationRequest.getComment(),
recategorizationRequest.getUser());
requiresReanalysis = true;
response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).build());
}
if (requiresReanalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
@Transactional
public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests) {
List<ManualAddResponse> response = new ArrayList<>();
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) {
var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
if (resizeRedactionRequest.getComment() != null) {
Long commentId = commentService.addCommentAndGetId(fileId,
resizeRedactionRequest.getAnnotationId(),
resizeRedactionRequest.getComment(),
resizeRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.updateDictionaryForResizeRedactions(dossierId,
fileId,
resizeRedaction,
getRedactionLogEntry(redactionLog, resizeRedaction.getId().getAnnotationId()));
resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(),
resizeRedaction.getId().getAnnotationId(),
resizeRedaction.getStatus(),
typeIdsOfModifiedDictionaries);
}
if (resizeRedactionRequests.stream().anyMatch(BaseManualRequest::isApproved)) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
@ -168,43 +311,12 @@ public class ManualRedactionService {
}
private Long addCommentAndGetId(String fileId, String annotationId, String comment, String user) {
if (comment == null) {
return null;
}
return addComment(fileId, annotationId, comment, user).getId();
}
private CommentEntity addComment(String fileId, String annotationId, String comment, String user) {
if (comment == null) {
return null;
}
return commentPersistenceService.insert(CommentEntity.builder()
.text(comment)
.fileId(fileId)
.annotationId(annotationId)
.user(user)
.date(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))
.build());
}
private void reprocess(String dossierId, String fileId) {
fileStatusService.setStatusReprocessForManual(dossierId, fileId, true);
}
public ManualRedactionEntryEntity getAddRedaction(String fileId, String annotationId) {
return addRedactionPersistenceService.findAddRedaction(fileId, annotationId);
}
private void removeFromDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) {
try {
@ -227,111 +339,12 @@ public class ManualRedactionService {
}
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId, List<RemoveRedactionRequest> removeRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId);
var requiresReAnalysis = false;
var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId);
//validate removing from dossier template dictionary
removeRedactionRequests.forEach(request -> {
if (request.isRemoveFromDictionary()) {
RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, request.getAnnotationId());
var dossierTemplateTypeId = toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(dossierTemplateTypeId,
request.isRemoveFromDictionary(),
request.isRemoveFromAllDossiers());
}
});
for (RemoveRedactionRequest removeRedactionRequest : removeRedactionRequests) {
if (manualAddRedactionsContains(manualRedactions, removeRedactionRequest.getAnnotationId()) && removeRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED)) {
log.info("hard delete ManualRedactions for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
manualRedactionProviderService.hardDeleteManualRedactions(fileId, removeRedactionRequest.getAnnotationId());
requiresReAnalysis = true;
continue;
}
log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
IdRemoval idRemoval = MagicConverter.convert(removeRedactionPersistenceService.insert(fileId, removeRedactionRequest), IdRemoval.class);
Long commentId = addCommentAndGetId(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser());
boolean matchingEntryFound = false;
if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) {
try {
getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId());
matchingEntryFound = true;
} catch (NotFoundException e) {
log.warn("No matching entry found in redaction log for annotation id {}", removeRedactionRequest.getAnnotationId());
}
requiresReAnalysis = requiresReAnalysis || matchingEntryFound;
}
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(//
getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()),//
dossierId, //
dossier.getDossierTemplateId(),//
fileId, //
removeRedactionRequest);
boolean removedFromDictionary = typeIdsOfModifiedDictionaries != null;
removeRedactionPersistenceService.updateStatus(fileId,
removeRedactionRequest.getAnnotationId(),
removeRedactionRequest.getStatus(),
removedFromDictionary,
typeIdsOfModifiedDictionaries);
if (!matchingEntryFound && !removedFromDictionary && idRemoval.isApproved()) {
removeRedactionPersistenceService.markAsProcessed(idRemoval);
}
requiresReAnalysis = requiresReAnalysis || removedFromDictionary;
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
if (requiresReAnalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
private boolean manualAddRedactionsContains(ManualRedactions manualRedactions, String annotationId) {
return manualRedactions.getEntriesToAdd().stream().anyMatch(m -> annotationId.equals(m.getAnnotationId()));
}
private boolean revertRemoveFromDictionary(String value, DossierEntity dossier, String fileId, IdRemovalEntity idRemoval) {
String annotationId = idRemoval.getId().getAnnotationId();
if (!idRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
return false;
}
if (!idRemoval.isRemoveFromDictionary()) {
removeRedactionPersistenceService.updateStatus(fileId, annotationId, idRemoval.getStatus(), false, Collections.emptySet());
return false;
}
idRemoval.getTypeIdsOfModifiedDictionaries().forEach(changedTypeId -> addToDictionary(changedTypeId, value, dossier.getId(), fileId, DictionaryEntryType.ENTRY));
removeRedactionPersistenceService.updateStatus(fileId, annotationId, idRemoval.getStatus(), true, Collections.emptySet());
return true;
}
private RedactionLogEntry getRedactionLogEntry(RedactionLog redactionLog, String annotationId) {
return redactionLog.getRedactionLogEntry()
@ -342,355 +355,6 @@ public class ManualRedactionService {
}
@Transactional
public List<ManualAddResponse> addForceRedaction(String dossierId, String fileId, List<ForceRedactionRequest> forceRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var requiresReanalysis = false;
for (var forceRedactionRequest : forceRedactionRequests) {
forceRedactionPersistenceService.insert(fileId, forceRedactionRequest);
Long commentId = addCommentAndGetId(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser());
requiresReanalysis = requiresReanalysis || forceRedactionRequest.isApproved();
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
if (requiresReanalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
@Transactional
public List<ManualAddResponse> addLegalBasisChange(String dossierId, String fileId, List<LegalBasisChangeRequest> legalBasisChangeRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
for (var legalBasisChangeRequest : legalBasisChangeRequests) {
legalBasisChangePersistenceService.insert(fileId, legalBasisChangeRequest);
Long commentId = addCommentAndGetId(fileId, legalBasisChangeRequest.getAnnotationId(), legalBasisChangeRequest.getComment(), legalBasisChangeRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
@Transactional
public List<ManualAddResponse> addRecategorization(String dossierId, String fileId, List<RecategorizationRequest> recategorizationRequests) {
var response = new ArrayList<ManualAddResponse>();
var requiresReanalysis = false;
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossierId, fileId);
for (var recategorizationRequest : recategorizationRequests) {
validateDictionaries(recategorizationRequest);
recategorizationPersistenceService.insert(fileId, recategorizationRequest);
Set<String> typeIdsOfDictionariesWithAdd = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
recategorizationRequest.getAnnotationId(),
recategorizationRequest);
Set<String> typeIdsOfDictionariesWithDelete = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(//
getRedactionLogEntry(redactionLog, recategorizationRequest.getAnnotationId()),//
recategorizationRequest.getDossierId(),//
dossier.getDossierTemplateId(),//
fileId,//
recategorizationRequest);
recategorizationPersistenceService.updateStatus(fileId,
recategorizationRequest.getAnnotationId(),
AnnotationStatus.APPROVED,
typeIdsOfDictionariesWithAdd != null,
typeIdsOfDictionariesWithAdd,
typeIdsOfDictionariesWithDelete);
Long commentId = addCommentAndGetId(fileId, recategorizationRequest.getAnnotationId(), recategorizationRequest.getComment(), recategorizationRequest.getUser());
requiresReanalysis = requiresReanalysis || recategorizationRequest.getStatus().equals(AnnotationStatus.APPROVED);
response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).build());
}
if (requiresReanalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
public CommentEntity addComment(String dossierId, String fileId, String annotationId, CommentRequest commentRequest) {
CommentEntity createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser());
fileStatusPersistenceService.updateHasComments(fileId, true);
if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) {
reprocess(dossierId, fileId);
return createdComment;
}
fileStatusService.setStatusProcessing(fileId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.forEach(entry -> entry.getComments().add(MagicConverter.convert(createdComment, RedactionLogComment.class)));
fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog);
fileStatusService.setStatusProcessed(fileId);
return createdComment;
}
@Transactional
public void deleteAddRedaction(String dossierId, String fileId, List<String> annotationIds) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var requiresReanalysis = false;
for (String annotationId : annotationIds) {
ManualRedactionEntryEntity addRedaction = getAddRedaction(fileId, annotationId);
boolean dictionaryChanged = manualRedactionDictionaryUpdateHandler.revertAddToDictionary(fileId, dossier.getId(), addRedaction);
requiresReanalysis = requiresReanalysis || dictionaryChanged;
addRedactionPersistenceService.updateStatus(fileId, annotationId, AnnotationStatus.APPROVED, false, null);
addRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
if (requiresReanalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@Transactional
public void deleteRemoveRedaction(String dossierId, String fileId, List<String> annotationIds) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var requiresReanalysis = false;
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
for (String annotationId : annotationIds) {
IdRemovalEntity removeRedaction = getRemoveRedaction(fileId, annotationId);
String originalValue = getRedactionLogEntry(redactionLog, annotationId).getValue();
boolean dictionaryChanged = revertRemoveFromDictionary(originalValue, dossier, fileId, removeRedaction);
requiresReanalysis = requiresReanalysis || dictionaryChanged;
removeRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
if (requiresReanalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private IdRemovalEntity getRemoveRedaction(String fileId, String annotationId) {
return removeRedactionPersistenceService.findRemoveRedaction(fileId, annotationId);
}
@Transactional
public void deleteForceRedaction(String dossierId, String fileId, List<String> annotationIds) {
var now = OffsetDateTime.now();
for (var annotationId : annotationIds) {
forceRedactionPersistenceService.softDelete(fileId, annotationId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@Transactional
public void deleteLegalBasisChange(String dossierId, String fileId, List<String> annotationIds) {
for (var annotationId : annotationIds) {
legalBasisChangePersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@Transactional
public void deleteRecategorization(String dossierId, String fileId, List<String> annotationIds) {
var requiresReanalysis = false;
for (var annotationId : annotationIds) {
var imageRecategorization = getRecategorization(fileId, annotationId);
recategorizationPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
requiresReanalysis = requiresReanalysis || !AnnotationStatus.REQUESTED.equals(imageRecategorization.getStatus());
}
if (requiresReanalysis) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private ManualImageRecategorizationEntity getRecategorization(String fileId, String annotationId) {
return recategorizationPersistenceService.findRecategorization(fileId, annotationId);
}
@Transactional
public void deleteComment(String dossierId, String fileId, List<Long> commentIds) {
for (var commentId : commentIds) {
commentPersistenceService.softDelete(commentId, OffsetDateTime.now());
}
// update indicator
fileStatusPersistenceService.updateHasComments(fileId, commentPersistenceService.fileHasComments(fileId));
// if file is currently not analyzing, we can quickly change the redaction log, else we must wait for the analysis to finish.
if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) {
reprocess(dossierId, fileId);
return;
}
fileStatusService.setStatusProcessing(fileId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
redactionLog.getRedactionLogEntry().forEach(entry -> entry.getComments().removeIf(comment -> commentIds.contains(comment.getId())));
fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog);
fileStatusService.setStatusProcessed(fileId);
}
@Transactional
public void deleteResizeRedaction(String dossierId, String fileId, List<String> annotationIds) {
OffsetDateTime now = OffsetDateTime.now();
for (var annotationId : annotationIds) {
resizeRedactionPersistenceService.softDelete(fileId, annotationId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@Transactional
public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests) {
List<ManualAddResponse> response = new ArrayList<>();
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) {
var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
if (resizeRedactionRequest.getComment() != null) {
Long commentId = addCommentAndGetId(fileId, resizeRedactionRequest.getAnnotationId(), resizeRedactionRequest.getComment(), resizeRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
updateDictionaryForResizeRedactions(dossierId, fileId, resizeRedaction, redactionLog);
}
if (resizeRedactionRequests.stream().anyMatch(BaseManualRequest::isApproved)) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
private void updateDictionaryForResizeRedactions(String dossierId, String fileId, ManualResizeRedactionEntity resizeRedaction, RedactionLog redactionLog) {
RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, resizeRedaction.getId().getAnnotationId());
if (resizeRedaction.getUpdateDictionary() != null && resizeRedaction.getUpdateDictionary() && resizeRedaction.getStatus()
.equals(AnnotationStatus.APPROVED) && (redactionLogEntry.isDictionaryEntry() || redactionLogEntry.isDossierDictionaryEntry())) {
var dossier = dossierPersistenceService.findByDossierId(dossierId);
var typeId = buildTypeId(redactionLogEntry, resizeRedaction, dossier);
var newValue = resizeRedaction.getValue();
var oldValue = redactionLogEntry.getValue();
var dictionaryEntryType = getDictionaryEntryType(redactionLogEntry);
boolean isShrinking = oldValue != null && oldValue.length() > newValue.length();
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
if (isShrinking) {
log.info("Remove old value '{}' from dictionary", oldValue);
removeFromDictionary(typeId, oldValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeId);
if (resizeRedaction.isAddToAllDossiers() && redactionLogEntry.isDictionaryEntry()) {
String dossierTemplateId = dossier.getDossierTemplateId();
var dossiersOfThisDossierTemplate = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
var type = redactionLogEntry.getType();
dossiersOfThisDossierTemplate.forEach(dossierEntity -> {
var typeIdOfDossierEntity = toTypeId(type, dossierTemplateId, dossierEntity.getId());
removeFromDictionary(typeIdOfDossierEntity, oldValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeIdOfDossierEntity);
});
}
}
log.info("Add new value '{}' to dictionary", newValue);
addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeId);
resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(),
resizeRedaction.getId().getAnnotationId(),
resizeRedaction.getStatus(),
typeIdsOfModifiedDictionaries);
}
}
private String buildTypeId(RedactionLogEntry redactionLogEntry, ManualResizeRedactionEntity resizeRedaction, DossierEntity dossier) {
if (resizeRedaction.isAddToAllDossiers()) {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
} else {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
}
}
private DictionaryEntryType getDictionaryEntryType(RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.isRecommendation() && redactionLogEntry.isFalsePositive()) {
return DictionaryEntryType.FALSE_RECOMMENDATION;
} else if (redactionLogEntry.isFalsePositive()) {
return DictionaryEntryType.FALSE_POSITIVE;
} else {
return DictionaryEntryType.ENTRY;
}
}
public ManualRedactions getManualRedactions(String fileId) {
return manualRedactionProviderService.getManualRedactions(fileId);
@ -740,13 +404,4 @@ public class ManualRedactionService {
}
}
public Comments getComments(String fileId) {
return new Comments(commentPersistenceService.findCommentsByFileID(fileId, false)
.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> MagicConverter.convert(entry.getValue(), Comment.class))));
}
}

153
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionUndoService.java
View File

@ -4,7 +4,9 @@ import static com.iqser.red.service.persistence.service.v1.api.external.resource
import static com.iqser.red.service.persistence.service.v1.api.external.resource.ManualRedactionResource.DOSSIER_ID;
import static com.iqser.red.service.persistence.service.v1.api.external.resource.ManualRedactionResource.FILE_ID;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
@ -13,11 +15,25 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@ -26,24 +42,41 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ManualRedactionWrapperModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualRedactionUndoService {
private final ManualRedactionService manualRedactionService;
private final AuditPersistenceService auditPersistenceService;
private final FileStatusService fileStatusService;
ManualRedactionProviderService manualRedactionProviderService;
AuditPersistenceService auditPersistenceService;
FileStatusService fileStatusService;
AnalysisFlagsCalculationService analysisFlagsCalculationService;
ImageRecategorizationPersistenceService recategorizationPersistenceService;
DossierPersistenceService dossierPersistenceService;
AddRedactionPersistenceService addRedactionPersistenceService;
RemoveRedactionPersistenceService removeRedactionPersistenceService;
ForceRedactionPersistenceService forceRedactionPersistenceService;
LegalBasisChangePersistenceService legalBasisChangePersistenceService;
ResizeRedactionPersistenceService resizeRedactionPersistenceService;
RedactionLogService redactionLogService;
ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler;
@Transactional
public void undo(String dossierId, String fileId, Set<String> annotationIds) {
// undo the latest manual redaction for each annotationId
ManualRedactions manualRedactions = manualRedactionService.getManualRedactions(fileId);
ManualRedactions manualRedactions = getManualRedactions(fileId);
Map<String, ManualRedactionWrapperModel> manualRedactionWrappers = getLatestManualRedactionsForAnnotationIds(manualRedactions, annotationIds);
@ -60,6 +93,13 @@ public class ManualRedactionUndoService {
reprocess(dossierId, fileId);
}
private ManualRedactions getManualRedactions(String fileId) {
return manualRedactionProviderService.getManualRedactions(fileId);
}
private void reprocess(String dossierId, String fileId) {
fileStatusService.setStatusReprocessForManual(dossierId, fileId, true);
@ -74,7 +114,7 @@ public class ManualRedactionUndoService {
.map(ManualRedactionWrapperModel::getId)
.collect(Collectors.toList());
if (!manualResizeRedactions.isEmpty()) {
manualRedactionService.deleteResizeRedaction(dossierId, fileId, manualResizeRedactions);
deleteResizeRedaction(dossierId, fileId, manualResizeRedactions);
manualResizeRedactions.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
@ -86,6 +126,16 @@ public class ManualRedactionUndoService {
}
private void deleteResizeRedaction(String dossierId, String fileId, List<String> annotationIds) {
OffsetDateTime now = OffsetDateTime.now();
for (var annotationId : annotationIds) {
resizeRedactionPersistenceService.softDelete(fileId, annotationId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private void undoLegalBasisChange(String dossierId, String fileId, Map<String, ManualRedactionWrapperModel> manualRedactionWrappers) {
List<String> manualLegalBasisChanges = manualRedactionWrappers.values()
@ -95,7 +145,7 @@ public class ManualRedactionUndoService {
.collect(Collectors.toList());
if (!manualLegalBasisChanges.isEmpty()) {
manualRedactionService.deleteLegalBasisChange(dossierId, fileId, manualLegalBasisChanges);
deleteLegalBasisChange(dossierId, fileId, manualLegalBasisChanges);
manualLegalBasisChanges.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
@ -107,6 +157,16 @@ public class ManualRedactionUndoService {
}
private void deleteLegalBasisChange(String dossierId, String fileId, List<String> annotationIds) {
for (var annotationId : annotationIds) {
legalBasisChangePersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private void undoRecategorization(String dossierId, String fileId, Map<String, ManualRedactionWrapperModel> manualRedactionWrappers) {
List<String> manualImageRecategorizations = manualRedactionWrappers.values()
@ -116,7 +176,7 @@ public class ManualRedactionUndoService {
.collect(Collectors.toList());
if (!manualImageRecategorizations.isEmpty()) {
manualRedactionService.deleteRecategorization(dossierId, fileId, manualImageRecategorizations);
deleteRecategorization(dossierId, fileId, manualImageRecategorizations);
manualImageRecategorizations.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
@ -128,6 +188,22 @@ public class ManualRedactionUndoService {
}
private void deleteRecategorization(String dossierId, String fileId, List<String> annotationIds) {
dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
for (var annotationId : annotationIds) {
ManualImageRecategorizationEntity recategorizationEntity = recategorizationPersistenceService.findRecategorization(fileId, annotationId);
String originalValue = getRedactionLogEntry(redactionLog, annotationId).getValue();
manualRedactionDictionaryUpdateHandler.revertRemoveFromDictionary(originalValue, dossierId, fileId, recategorizationEntity.getTypeIdsOfDictionariesWithDelete());
manualRedactionDictionaryUpdateHandler.revertAddToDictionary(originalValue, DictionaryEntryType.ENTRY, fileId, dossierId, recategorizationEntity.getTypeIdsOfDictionariesWithAdd());
recategorizationPersistenceService.updateStatus(fileId, annotationId, AnnotationStatus.APPROVED, false, Collections.emptySet(), Collections.emptySet());
recategorizationPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
} analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private void undoForceRedactions(String dossierId, String fileId, Map<String, ManualRedactionWrapperModel> manualRedactionWrappers) {
List<String> manualForceRedactions = manualRedactionWrappers.values()
@ -137,7 +213,7 @@ public class ManualRedactionUndoService {
.collect(Collectors.toList());
if (!manualForceRedactions.isEmpty()) {
manualRedactionService.deleteForceRedaction(dossierId, fileId, manualForceRedactions);
deleteForceRedaction(dossierId, fileId, manualForceRedactions);
manualForceRedactions.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
@ -149,6 +225,18 @@ public class ManualRedactionUndoService {
}
private void deleteForceRedaction(String dossierId, String fileId, List<String> annotationIds) {
var now = OffsetDateTime.now();
for (var annotationId : annotationIds) {
forceRedactionPersistenceService.softDelete(fileId, annotationId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private void undoIdRemovals(String dossierId, String fileId, Map<String, ManualRedactionWrapperModel> manualRedactionWrappers) {
List<String> idRemovals = manualRedactionWrappers.values()
@ -157,7 +245,7 @@ public class ManualRedactionUndoService {
.map(ManualRedactionWrapperModel::getId)
.collect(Collectors.toList());
if (!idRemovals.isEmpty()) {
manualRedactionService.deleteRemoveRedaction(dossierId, fileId, idRemovals);
deleteRemoveRedaction(dossierId, fileId, idRemovals);
idRemovals.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
@ -169,6 +257,24 @@ public class ManualRedactionUndoService {
}
private void deleteRemoveRedaction(String dossierId, String fileId, List<String> annotationIds) {
dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
for (String annotationId : annotationIds) {
IdRemovalEntity removeRedaction = removeRedactionPersistenceService.findRemoveRedaction(fileId, annotationId);
String originalValue = getRedactionLogEntry(redactionLog, annotationId).getValue();
boolean dictionaryChanged = manualRedactionDictionaryUpdateHandler.revertRemoveFromDictionary(originalValue, dossierId, fileId, removeRedaction);
removeRedactionPersistenceService.updateStatus(fileId, annotationId, removeRedaction.getStatus(), dictionaryChanged, Collections.emptySet());
removeRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private void undoManualRedactionEntries(String dossierId, String fileId, Map<String, ManualRedactionWrapperModel> manualRedactionWrappers) {
List<String> manualRedactionEntries = manualRedactionWrappers.values()
@ -177,7 +283,7 @@ public class ManualRedactionUndoService {
.map(ManualRedactionWrapperModel::getId)
.collect(Collectors.toList());
if (!manualRedactionEntries.isEmpty()) {
manualRedactionService.deleteAddRedaction(dossierId, fileId, manualRedactionEntries);
deleteAddRedaction(dossierId, fileId, manualRedactionEntries);
manualRedactionEntries.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
@ -189,6 +295,33 @@ public class ManualRedactionUndoService {
}
private void deleteAddRedaction(String dossierId, String fileId, List<String> annotationIds) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
for (String annotationId : annotationIds) {
ManualRedactionEntryEntity addRedaction = addRedactionPersistenceService.findAddRedaction(fileId, annotationId);
boolean dictionaryChanged = manualRedactionDictionaryUpdateHandler.revertAddToDictionary(fileId, dossier.getId(), addRedaction);
addRedactionPersistenceService.updateStatus(fileId, annotationId, AnnotationStatus.APPROVED, dictionaryChanged, null);
addRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private RedactionLogEntry getRedactionLogEntry(RedactionLog redactionLog, String annotationId) {
return redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.findFirst()
.orElseThrow(() -> new NotFoundException("Annotation does not exist in redaction log."));
}
private Map<String, ManualRedactionWrapperModel> getLatestManualRedactionsForAnnotationIds(ManualRedactions manualRedactions, Set<String> annotationIds) {
Map<String, ManualRedactionWrapperModel> result = new HashMap<>();

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java
View File

@ -33,6 +33,7 @@ public class ImageRecategorizationPersistenceService {
manualImageRecategorization.setId(new AnnotationEntityId(recategorizationRequest.getAnnotationId(), fileId));
BeanUtils.copyProperties(recategorizationRequest, manualImageRecategorization);
manualImageRecategorization.setRequestDate(OffsetDateTime.now());
manualImageRecategorization.setTypeId(recategorizationRequest.getDossierTemplateTypeId());
imageRecategorizationRepository.saveAndFlush(manualImageRecategorization);
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/ManualImageRecategorizationMapper.java
View File

@ -8,9 +8,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
public class ManualImageRecategorizationMapper implements BiConsumer<ManualImageRecategorizationEntity, ManualImageRecategorization> {
@Override
public void accept(ManualImageRecategorizationEntity manualRedactionEntryEntity, ManualImageRecategorization manualRedactionEntry) {
public void accept(ManualImageRecategorizationEntity manualImageRecategorizationEntity, ManualImageRecategorization manualRedactionEntry) {
manualRedactionEntry.setType(manualRedactionEntryEntity.getTypeId().substring(0, manualRedactionEntryEntity.getTypeId().indexOf(":")));
manualRedactionEntry.setType(manualImageRecategorizationEntity.getTypeId().substring(0, manualImageRecategorizationEntity.getTypeId().indexOf(":")));
}
}

13
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java
View File

@ -2,8 +2,6 @@ package com.iqser.red.service.peristence.v1.server.integration.service;
import static org.assertj.core.api.Assertions.assertThat;
import java.awt.Color;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -11,9 +9,7 @@ import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryC
import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
@Service
public class TypeProvider {
@ -21,6 +17,7 @@ public class TypeProvider {
@Autowired
private DictionaryClient dictionaryClient;
public TypeValue testAndProvideType(DossierTemplateModel dossierTemplate) {
return testAndProvideType(dossierTemplate, null, "test", false);
@ -32,9 +29,13 @@ public class TypeProvider {
return testAndProvideType(dossierTemplate, dossier, typeName, false);
}
public TypeValue testAndProvideType(DossierTemplateModel dossierTemplate, Dossier dossier, String typeName, boolean dossierDictionaryOnly) {
return testAndProvideType(dossierTemplate, dossier, typeName, dossierDictionaryOnly, 100);
}
public TypeValue testAndProvideType(DossierTemplateModel dossierTemplate, Dossier dossier, String typeName, boolean dossierDictionaryOnly, int rank) {
var type = new CreateTypeValue();
@ -53,9 +54,9 @@ public class TypeProvider {
type.setDossierDictionaryOnly(dossierDictionaryOnly);
dictionaryClient.addType(type);
var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getDossierTemplateId(),dossier != null ? dossier.getId() : null,false);
var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getDossierTemplateId(), dossier != null ? dossier.getId() : null, false);
var foundType =allTypes.getTypes().stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findAny();
var foundType = allTypes.getTypes().stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findAny();
assertThat(foundType).isPresent();
return foundType.get();

126
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -1,32 +1,7 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import com.google.common.collect.Sets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.peristence.v1.server.integration.client.*;
import com.iqser.red.service.peristence.v1.server.integration.service.*;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.*;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
import feign.FeignException;
import lombok.SneakyThrows;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
@ -36,7 +11,58 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import com.google.common.collect.Sets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileManagementClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ReanalysisClient;
import com.iqser.red.service.peristence.v1.server.integration.client.RedactionLogClient;
import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ViewedPagesClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageExclusionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ViewedPagesRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import feign.FeignException;
import lombok.SneakyThrows;
public class FileTest extends AbstractPersistenceServerServiceTest {
@ -103,7 +129,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
fileManagementClient.deleteFile(dossier.getId(), file.getId());
var nrOfFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId()).size();
@ -330,31 +355,37 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
var annotationId = "imagine_this_makes_sense";
RedactionLog redactionLog = new RedactionLog();
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder().type(type.getType()).id(annotationId).build();
redactionLog.getRedactionLogEntry().add(redactionLogEntry);
when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(redactionLog);
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
var addRedaction = manualRedactionClient.addRedactionBulk(dossierId,
fileId,
Set.of(AddRedactionRequestModel.builder()
.addToDictionary(true)
.type(type.getType())
.addToAllDossiers(true)
.reason("1")
.value("test")
.legalBasis("1")
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.build())).iterator().next();
var addRedactionRequest = AddRedactionRequestModel.builder()
.addToDictionary(true)
.type(type.getType())
.addToAllDossiers(true)
.reason("1")
.value("test")
.legalBasis("1")
.build();
manualRedactionClient.addRedactionBulk(dossierId, fileId, Set.of(addRedactionRequest));
manualRedactionClient.removeRedactionBulk(dossierId,
fileId,
Set.of(RemoveRedactionRequestModel.builder().annotationId(addRedaction.getAnnotationId()).comment("comment").removeFromDictionary(false).build()));
Set.of(RemoveRedactionRequestModel.builder().annotationId(annotationId).comment("comment").removeFromDictionary(false).build()));
manualRedactionClient.forceRedactionBulk(dossierId,
fileId,
Set.of(ForceRedactionRequestModel.builder().annotationId("forceRedactionAnnotation").comment("comment").legalBasis("1").build()));
manualRedactionClient.legalBasisChangeBulk(dossierId,
manualRedactionClient.legalBasisChangeBulk(dossierId,
fileId,
Set.of(LegalBasisChangeRequestModel.builder().annotationId("legalBasisChangeAnnotation").comment("comment").legalBasis("1").build()));
manualRedactionClient.recategorizeBulk(dossierId,
fileId,
Set.of(RecategorizationRequestModel.builder().annotationId("imageRecategorizationAnnotation").comment("comment").type("new-type").build()));
Set.of(RecategorizationRequestModel.builder().annotationId(annotationId).comment("comment").type("new-type").build()));
var loadedFile = fileClient.getFileStatus(dossierId, fileId);
@ -409,12 +440,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
.reason("1")
.value("test")
.legalBasis("1")
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.build())).iterator().next();
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
reanalysisClient.toggleExclusion(dossier.getId(), file.getId(), true);
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.isExcluded()).isTrue();
@ -438,7 +467,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier, fileName);
String fileId = file.getId();
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
@ -515,6 +543,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
}
@Test
public void testFile6034SetUnderReview() {
@ -542,17 +571,15 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(actualMessage).contains(expectedMessage);
exception = Assertions.assertThrows(FeignException.BadRequest.class, () -> {
fileClient.setStatusUnderReview(dossier.getId(), file.getId(), user2);
});
expectedMessage = "User must be dossier member";
actualMessage = exception.getMessage();
actualMessage = exception.getMessage();
assertThat(actualMessage).contains(expectedMessage);
fileClient.setStatusUnderReview(dossier.getId(), file.getId(), altUserId);
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);
@ -561,6 +588,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(loadedFile.getLastApprover()).isNull();
}
@Test
@SneakyThrows
public void testUploadFileIsInOcrProcessingWithOcrByDefaultFlagTrue() {

77
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -13,10 +13,12 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Redacti
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.Dictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ResizeRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@ -238,10 +240,10 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("Luke Skywalker"), false, dossier.getDossierId(), DictionaryEntryType.ENTRY);
var dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
Dictionary dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder("Luke Skywalker", "Darth Vader");
var dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId());
Dictionary dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId());
assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrder("Luke Skywalker");
var redactionLog = new RedactionLog(1,
@ -258,7 +260,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(),
Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).removeFromAllDossiers(true).build())).get(0);
Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).removeFromAllDossiers(true).build()));
dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder("Darth Vader");
@ -948,4 +950,73 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertThat(mergedDictForTypeDosTempDictInDossier2.getEntries().get(0)).isEqualTo("test redaction in dossier template");
}
@Test
public void testManualRecategorizeAndUndo() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
var type = typeProvider.testAndProvideType(dossierTemplate);
var type2 = typeProvider.testAndProvideType(dossierTemplate, dossier, "test2", false, 66);
var entries = new ArrayList<String>();
var lukeSkywalker = "Luke Skywalker";
var darthVader = "Darth Vader";
entries.add(lukeSkywalker);
entries.add(darthVader);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), entries, false, null, DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of(lukeSkywalker), false, dossier.getDossierId(), DictionaryEntryType.ENTRY);
Dictionary dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder(lukeSkywalker, darthVader);
Dictionary dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId());
assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrder(lukeSkywalker);
Dictionary dossierTemplateDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), null);
assertThat(dossierTemplateDictionary2.getEntries()).isEmpty();
Dictionary dossierDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), dossier.getDossierId());
assertThat(dossierDictionary2.getEntries()).isEmpty();
var annotationId = "AnnotationId";
var redactionLog = new RedactionLog(1,
1,
List.of(RedactionLogEntry.builder().id(annotationId).type(type.getType()).value(lukeSkywalker).isDictionaryEntry(true).build()),
null,
0,
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(redactionLog);
manualRedactionClient.recategorizeBulk(dossier.getId(),
file.getId(),
Set.of(RecategorizationRequestModel.builder().type(type2.getType()).annotationId(annotationId).addToDictionary(true).addToAllDossiers(true).build()));
dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder(darthVader);
dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId());
assertThat(dossierDictionary.getEntries()).isEmpty();
dossierTemplateDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), null);
assertThat(dossierTemplateDictionary2.getEntries()).containsExactlyInAnyOrder(lukeSkywalker);
dossierDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), dossier.getDossierId());
assertThat(dossierDictionary2.getEntries()).containsExactlyInAnyOrder(lukeSkywalker);
manualRedactionClient.undo(dossier.getDossierId(), file.getFileId(), Set.of(annotationId));
dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder(lukeSkywalker, darthVader);
dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId());
assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrder(lukeSkywalker);
dossierTemplateDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), null);
assertThat(dossierTemplateDictionary2.getEntries()).isEmpty();
dossierDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), dossier.getDossierId());
assertThat(dossierDictionary2.getEntries()).isEmpty();
}
}

11
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRequestWithAddToDictionary.java
View File

@ -4,12 +4,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
public interface ManualRequestWithAddToDictionary extends BaseManualRequest {
String getValue();
String getDossierId();
boolean isAddToDictionary();
boolean isAddToAllDossiers();
boolean isForceAddToDictionary();
String getDossierTemplateTypeId();
DictionaryEntryType getDictionaryEntryType();
}

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RecategorizationRequest.java
View File

@ -23,7 +23,6 @@ public class RecategorizationRequest implements ManualRequestWithAddToDictionary
String dossierId;
String comment;
int page;
String value;
boolean addToDictionary;
boolean addToAllDossiers;
private DictionaryEntryType dictionaryEntryType;