RED-8480: added some checks and tests

This commit is contained in:
Ali Oezyetimoglu 2024-02-15 22:05:06 +01:00
parent d2b6971f04
commit e681651420
5 changed files with 981 additions and 673 deletions

View File

@ -9,6 +9,7 @@ import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -142,6 +143,7 @@ public class ManualRedactionMapper {
EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId, Collections.emptyList(), includeUnprocessed);
List<RecategorizationRequest> requests = new ArrayList<>();
for (RecategorizationRequestModel recategorizationRequest : recategorizationRequests) {
checkLegalBasisIsNotNull(recategorizationRequest.getLegalBasis());
EntityLogEntry entityLogEntry = getEntityLogEntry(entityLog, recategorizationRequest.getAnnotationId());
RecategorizationRequest build = RecategorizationRequest.builder()
.annotationId(recategorizationRequest.getAnnotationId())
@ -164,6 +166,13 @@ public class ManualRedactionMapper {
}
private void checkLegalBasisIsNotNull(String legalBasis) {
if(legalBasis == null) {
throw new BadRequestException("No legal basis provided.");
}
}
public List<ResizeRedactionRequest> toResizeRedactionRequestList(Set<ResizeRedactionRequestModel> resizeRedactionRequests) {
return resizeRedactionRequests.stream()

View File

@ -75,6 +75,8 @@ public class ManualRedactionService {
EntityLogMergeService entityLogMergeService;
FileStatusPersistenceService fileStatusPersistenceService;
private static final int MAX_LEGAL_BASIS_LENGTH = 4000;
@Transactional
@Observed(name = "ManualRedactionService", contextualName = "add-manual-redaction")
@ -94,14 +96,15 @@ public class ManualRedactionService {
manualRedactionEntryEntities.add(addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest));
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
addRedactionRequest.getValue(),
addRedactionRequest);
addRedactionRequest.getValue(),
addRedactionRequest);
addRedactionPersistenceService.updateModifiedDictionaries(fileId, annotationId, !typeIdsOfModifiedDictionaries.isEmpty(), typeIdsOfModifiedDictionaries);
Long commentId = commentService.addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId)
.build());
}
manualRedactionEntryEntities = manualRedactionEntryEntities.stream()
@ -110,7 +113,8 @@ public class ManualRedactionService {
FileModel fileStatus = fileStatusService.getStatus(fileId);
if (!manualRedactionEntryEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(manualRedactionEntryEntities)).build();
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(manualRedactionEntryEntities))
.build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions);
} else {
reprocess(dossierId, fileId);
@ -123,7 +127,9 @@ public class ManualRedactionService {
private Set<ManualRedactionEntry> convertEntriesToAdd(List<ManualRedactionEntryEntity> source) {
return source.stream().map(entry -> convert(entry, ManualRedactionEntry.class, new ManualRedactionMapper())).collect(Collectors.toSet());
return source.stream()
.map(entry -> convert(entry, ManualRedactionEntry.class, new ManualRedactionMapper()))
.collect(Collectors.toSet());
}
@ -135,27 +141,25 @@ public class ManualRedactionService {
//validate removing from dossier template dictionary
for (RemoveRedactionRequest removeRedactionRequest : removeRedactionRequests) {
manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest,
removeRedactionRequest.getTypeToRemove(),
removeRedactionRequest.getDossierTemplateId());
removeRedactionRequest.getTypeToRemove(),
removeRedactionRequest.getDossierTemplateId());
log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
removeRedactionPersistenceService.insert(fileId, removeRedactionRequest);
Long commentId = commentService.addCommentAndGetId(fileId,
removeRedactionRequest.getAnnotationId(),
removeRedactionRequest.getComment(),
removeRedactionRequest.getUser());
removeRedactionRequest.getAnnotationId(),
removeRedactionRequest.getComment(),
removeRedactionRequest.getUser());
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(fileId, removeRedactionRequest);
boolean removedFromDictionary = !typeIdsOfModifiedDictionaries.isEmpty();
removeRedactionPersistenceService.updateModifiedDictionaries(fileId,
removeRedactionRequest.getAnnotationId(),
removedFromDictionary,
typeIdsOfModifiedDictionaries);
removeRedactionPersistenceService.updateModifiedDictionaries(fileId, removeRedactionRequest.getAnnotationId(), removedFromDictionary, typeIdsOfModifiedDictionaries);
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId)
.build());
}
reprocess(dossierId, fileId);
@ -176,11 +180,12 @@ public class ManualRedactionService {
forceRedactionPersistenceService.insert(fileId, forceRedactionRequest);
Long commentId = commentService.addCommentAndGetId(fileId,
forceRedactionRequest.getAnnotationId(),
forceRedactionRequest.getComment(),
forceRedactionRequest.getUser());
forceRedactionRequest.getAnnotationId(),
forceRedactionRequest.getComment(),
forceRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId)
.build());
}
reprocess(dossierId, fileId);
@ -201,11 +206,12 @@ public class ManualRedactionService {
legalBasisChangePersistenceService.insert(fileId, legalBasisChangeRequest);
Long commentId = commentService.addCommentAndGetId(fileId,
legalBasisChangeRequest.getAnnotationId(),
legalBasisChangeRequest.getComment(),
legalBasisChangeRequest.getUser());
legalBasisChangeRequest.getAnnotationId(),
legalBasisChangeRequest.getComment(),
legalBasisChangeRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId)
.build());
}
reprocess(dossierId, fileId);
fileStatusPersistenceService.setLastManualChangeDate(fileId, OffsetDateTime.now());
@ -221,28 +227,31 @@ public class ManualRedactionService {
for (var recategorizationRequest : recategorizationRequests) {
manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(recategorizationRequest, recategorizationRequest.getValue());
manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(recategorizationRequest,
recategorizationRequest.getTypeToRemove(),
recategorizationRequest.getDossierTemplateId());
recategorizationRequest.getTypeToRemove(),
recategorizationRequest.getDossierTemplateId());
checkLegalBasisLength(recategorizationRequest.getLegalBasis());
recategorizationPersistenceService.insert(fileId, recategorizationRequest);
Set<String> typeIdsOfDictionariesWithAdd = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
recategorizationRequest.getValue(),
recategorizationRequest);
recategorizationRequest.getValue(),
recategorizationRequest);
Set<String> typeIdsOfDictionariesWithDelete = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(fileId,
recategorizationRequest);
recategorizationRequest);
recategorizationPersistenceService.updateModifiedDictionaries(fileId,
recategorizationRequest.getAnnotationId(),
typeIdsOfDictionariesWithAdd,
typeIdsOfDictionariesWithDelete);
recategorizationRequest.getAnnotationId(),
typeIdsOfDictionariesWithAdd,
typeIdsOfDictionariesWithDelete);
Long commentId = commentService.addCommentAndGetId(fileId,
recategorizationRequest.getAnnotationId(),
recategorizationRequest.getComment(),
recategorizationRequest.getUser());
recategorizationRequest.getAnnotationId(),
recategorizationRequest.getComment(),
recategorizationRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId)
.build());
}
reprocess(dossierId, fileId);
@ -252,6 +261,14 @@ public class ManualRedactionService {
}
private void checkLegalBasisLength(String legalBasis) {
if (legalBasis.length() > MAX_LEGAL_BASIS_LENGTH) {
throw new BadRequestException(String.format("The legal basis is too long (%s), max length %s", legalBasis.length(), MAX_LEGAL_BASIS_LENGTH));
}
}
@Transactional
@Observed(name = "ManualRedactionService", contextualName = "add-manual-redaction")
public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests, boolean includeUnprocessed) {
@ -267,18 +284,23 @@ public class ManualRedactionService {
if (resizeRedactionRequest.getComment() != null) {
Long commentId = commentService.addCommentAndGetId(fileId,
resizeRedactionRequest.getAnnotationId(),
resizeRedactionRequest.getComment(),
resizeRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build());
resizeRedactionRequest.getAnnotationId(),
resizeRedactionRequest.getComment(),
resizeRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId)
.build());
}
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.updateDictionaryForResizeRedactions(dossierId,
fileId,
resizeRedaction,
getEntityLogEntry(entityLog, resizeRedaction.getId().getAnnotationId()));
fileId,
resizeRedaction,
getEntityLogEntry(entityLog,
resizeRedaction.getId()
.getAnnotationId()));
resizeRedactionPersistenceService.updateModifiedDictionaries(resizeRedaction.getId().getFileId(), resizeRedaction.getId().getAnnotationId(), typeIdsOfModifiedDictionaries);
resizeRedactionPersistenceService.updateModifiedDictionaries(resizeRedaction.getId().getFileId(),
resizeRedaction.getId().getAnnotationId(),
typeIdsOfModifiedDictionaries);
}
manualResizeRedactionEntities = manualResizeRedactionEntities.stream()
@ -287,7 +309,8 @@ public class ManualRedactionService {
FileModel fileStatus = fileStatusService.getStatus(fileId);
if (!manualResizeRedactionEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) {
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(convertResizeRedactions(manualResizeRedactionEntities)).build();
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(convertResizeRedactions(manualResizeRedactionEntities))
.build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierPersistenceService.getAndValidateDossier(dossierId), fileStatusService.getStatus(fileId), manualRedactions);
} else {
reprocess(dossierId, fileId);
@ -301,14 +324,20 @@ public class ManualRedactionService {
private Set<ManualResizeRedaction> convertResizeRedactions(List<ManualResizeRedactionEntity> source) {
return source.stream().map(entry -> convert(entry, ManualResizeRedaction.class, new ManualResizeRedactionMapper())).collect(Collectors.toSet());
return source.stream()
.map(entry -> convert(entry, ManualResizeRedaction.class, new ManualResizeRedactionMapper()))
.collect(Collectors.toSet());
}
private void validatePositions(String fileId, AddRedactionRequest addRedactionRequest) {
var numberOfPages = fileStatusService.getStatus(fileId).getNumberOfPages();
addRedactionRequest.getPositions().stream().filter(p -> p.getPage() > numberOfPages).findAny().ifPresent(p -> new BadRequestException("Invalid page found in the request"));
addRedactionRequest.getPositions()
.stream()
.filter(p -> p.getPage() > numberOfPages)
.findAny()
.ifPresent(p -> new BadRequestException("Invalid page found in the request"));
}
@ -340,46 +369,52 @@ public class ManualRedactionService {
if (manualRedactions != null) {
if (manualRedactions.getEntriesToAdd() != null) {
manualRedactions.getEntriesToAdd().forEach(e -> {
if (e.getProcessedDate() == null) {
addRedactionPersistenceService.markAsProcessed(e);
}
});
manualRedactions.getEntriesToAdd()
.forEach(e -> {
if (e.getProcessedDate() == null) {
addRedactionPersistenceService.markAsProcessed(e);
}
});
}
if (manualRedactions.getIdsToRemove() != null) {
manualRedactions.getIdsToRemove().forEach(e -> {
if (e.getProcessedDate() == null) {
removeRedactionPersistenceService.markAsProcessed(e);
}
});
manualRedactions.getIdsToRemove()
.forEach(e -> {
if (e.getProcessedDate() == null) {
removeRedactionPersistenceService.markAsProcessed(e);
}
});
}
if (manualRedactions.getForceRedactions() != null) {
manualRedactions.getForceRedactions().forEach(e -> {
if (e.getProcessedDate() == null) {
forceRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
}
});
manualRedactions.getForceRedactions()
.forEach(e -> {
if (e.getProcessedDate() == null) {
forceRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
}
});
}
if (manualRedactions.getRecategorizations() != null) {
manualRedactions.getRecategorizations().forEach(e -> {
if (e.getProcessedDate() == null) {
recategorizationPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
}
});
manualRedactions.getRecategorizations()
.forEach(e -> {
if (e.getProcessedDate() == null) {
recategorizationPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
}
});
}
if (manualRedactions.getResizeRedactions() != null) {
manualRedactions.getResizeRedactions().forEach(e -> {
if (e.getProcessedDate() == null) {
resizeRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
}
});
manualRedactions.getResizeRedactions()
.forEach(e -> {
if (e.getProcessedDate() == null) {
resizeRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
}
});
}
if (manualRedactions.getLegalBasisChanges() != null) {
manualRedactions.getLegalBasisChanges().forEach(e -> {
if (e.getProcessedDate() == null) {
legalBasisChangePersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
}
});
manualRedactions.getLegalBasisChanges()
.forEach(e -> {
if (e.getProcessedDate() == null) {
legalBasisChangePersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
}
});
}
}
}

View File

@ -28,21 +28,18 @@ public class RecategorizationPersistenceService {
public void insert(String fileId, RecategorizationRequest recategorizationRequest) {
ManualRecategorizationEntity manualImageRecategorization = new ManualRecategorizationEntity();
manualImageRecategorization.setId(new AnnotationEntityId(recategorizationRequest.getAnnotationId(), fileId));
BeanUtils.copyProperties(recategorizationRequest, manualImageRecategorization);
manualImageRecategorization.setRequestDate(OffsetDateTime.now());
manualImageRecategorization.setTypeId(recategorizationRequest.getDossierTemplateTypeId());
recategorizationRepository.saveAndFlush(manualImageRecategorization);
ManualRecategorizationEntity manualRecategorization = new ManualRecategorizationEntity();
manualRecategorization.setId(new AnnotationEntityId(recategorizationRequest.getAnnotationId(), fileId));
BeanUtils.copyProperties(recategorizationRequest, manualRecategorization);
manualRecategorization.setRequestDate(OffsetDateTime.now());
manualRecategorization.setTypeId(recategorizationRequest.getDossierTemplateTypeId());
recategorizationRepository.saveAndFlush(manualRecategorization);
}
@Transactional
public void updateModifiedDictionaries(String fileId,
String annotationId, Set<String> typeIdsOfDictionaryWithAdd,
Set<String> typeIdsOfDictionaryWithDelete) {
public void updateModifiedDictionaries(String fileId, String annotationId, Set<String> typeIdsOfDictionaryWithAdd, Set<String> typeIdsOfDictionaryWithDelete) {
ManualRecategorizationEntity addRedaction = recategorizationRepository.findById(new AnnotationEntityId(annotationId, fileId))
.orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));

View File

@ -416,7 +416,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
.build()));
manualRedactionClient.recategorizeBulk(dossierId,
fileId,
Set.of(RecategorizationRequestModel.builder().annotationId(annotationId).comment("comment").type("new-type")
Set.of(RecategorizationRequestModel.builder().annotationId(annotationId).comment("comment").type("new-type").legalBasis("")
.build()),
false);
@ -424,22 +424,22 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
fileManagementClient.deleteFile(dossier.getId(), file.getId());
var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
assertThat(softDeletedFiles.size()).isEqualTo(1);
assertThat(softDeletedFiles).hasSize(1);
var activeFiles = fileClient.getDossierStatus(dossier.getId());
assertThat(activeFiles.size()).isEqualTo(0);
assertThat(activeFiles).isEmpty();
fileManagementClient.restoreFiles(dossier.getId(), Sets.newHashSet(file.getId()));
softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
assertThat(softDeletedFiles.size()).isEqualTo(0);
assertThat(softDeletedFiles).isEmpty();
activeFiles = fileClient.getDossierStatus(dossier.getId());
assertThat(activeFiles.size()).isEqualTo(1);
assertThat(activeFiles).hasSize(1);
fileManagementClient.hardDeleteFiles(dossier.getId(), List.of(file.getId()));
softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
assertThat(softDeletedFiles.size()).isEqualTo(0);
assertThat(softDeletedFiles).isEmpty();
activeFiles = fileClient.getDossierStatus(dossier.getId());
assertThat(activeFiles.size()).isEqualTo(0);
assertThat(activeFiles).isEmpty();
}