Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-3839: Remove manual redactions instead of skipping

Browse Source
This commit is contained in:
Philipp Schramm 2022-04-19 15:48:46 +02:00
parent 03e35f8f45
commit 3cfd4ea31f
7 changed files with 255 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java
View File

@ -73,14 +73,20 @@ public class AddRedactionPersistenceService {
public ManualRedactionEntryEntity findAddRedaction(String fileId, String annotationId) {
return manualRedactionRepository.findAddRedaction(new AnnotationEntityId(annotationId, fileId)).orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
return manualRedactionRepository.findAddRedaction(new AnnotationEntityId(annotationId, fileId))
.orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
}
public Set<ManualRedactionEntryEntity> findAddRedactions(String fileId, boolean includeDeletions) {
return new HashSet<>(manualRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions));
}
public List<ManualRedactionEntryEntity> findAllAddRedactions() {
return manualRedactionRepository.findAll();
}

47
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java
View File

@ -1,23 +1,25 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class CommentPersistenceService {
private final CommentRepository commentRepository;
@ -29,40 +31,57 @@ public class CommentPersistenceService {
public CommentEntity findComment(long commentId) {
return commentRepository.findById(commentId).orElseThrow(() -> new NotFoundException("Comment with id: " + commentId + " not found"));
}
public List<CommentEntity> findCommentsByAnnotationId(String fileId, String annotationId, boolean includeDeletions) {
return commentRepository.findByFileIdAndAnnotationId(fileId, annotationId, includeDeletions);
}
public Map<String, List<CommentEntity>> findCommentsByFileID(String fileId, boolean includeDeletions) {
List<CommentEntity> comments = commentRepository.findByFileId(fileId, includeDeletions);
return comments.stream().sorted(Comparator.comparing(CommentEntity::getDate)).collect(Collectors.groupingBy(CommentEntity::getAnnotationId));
}
public boolean fileHasComments(String fileId) {
return commentRepository.existsByFileIdAndSoftDeletedTimeIsNull(fileId);
}
@Transactional
public void hardDelete(long commentId) {
commentRepository.deleteById(commentId);
}
@Transactional
public void hardDelete(String fileId, String annotationId) {
List<CommentEntity> comments = findCommentsByAnnotationId(fileId, annotationId, true);
commentRepository.deleteAllInBatch(comments);
}
public List<CommentEntity> findCommentsByAnnotationId(String fileId, String annotationId, boolean includeDeletions) {
return commentRepository.findByFileIdAndAnnotationId(fileId, annotationId, includeDeletions);
}
@Transactional
public void softDelete(long commentId, OffsetDateTime softDeletedTime) {
commentRepository.updateSoftDelete(commentId, softDeletedTime);
}
@Transactional
public void undelete(long commentId) {
commentRepository.updateSoftDelete(commentId, null);
}
}

35
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java
View File

@ -1,21 +1,25 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RemoveRedactionRepository;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
@Service
@ -40,20 +44,29 @@ public class RemoveRedactionPersistenceService {
public IdRemovalEntity findRemoveRedaction(String fileId, String annotationId) {
return removeRedactionRepository.findByIdAndNotSoftDeleted(new AnnotationEntityId(annotationId, fileId))
.orElseThrow(() ->
new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
.orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
}
public Set<IdRemovalEntity> findRemoveRedactions(String fileId, boolean includeDeletions) {
return removeRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions).stream().collect(Collectors.toSet());
}
public List<IdRemovalEntity> findAllRemoveRedactions() {
return removeRedactionRepository.findAll();
}
@Transactional
public void hardDelete(String fileId, String annotationId) {
removeRedactionRepository.deleteById(new AnnotationEntityId(annotationId, fileId));
}
@Transactional
public void softDelete(String fileId, String annotationId, OffsetDateTime softDeleteTime) {

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java
View File

@ -6,6 +6,7 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.migration.migrations.DeleteRemovedManualAddRedactions7;
import com.iqser.red.service.peristence.v1.server.migration.migrations.DictionaryToEntityMigration2;
import com.iqser.red.service.peristence.v1.server.migration.migrations.EntityTypesMigration4;
import com.iqser.red.service.peristence.v1.server.migration.migrations.IndexMigration1;
@ -33,6 +34,7 @@ public class MigrationStarterService {
private final EntityTypesMigration4 entityTypesMigration4;
private final TypeToEntityMigration5 typeToEntityMigration5;
private final RemoveFalsePositiveManualRedactions6 removeFalsePositiveManualRedactions6;
private final DeleteRemovedManualAddRedactions7 deleteRemovedManualAddRedactions7;
private final FileManagementServiceSettings settings;
private final ApplicationContext ctx;
@ -55,6 +57,7 @@ public class MigrationStarterService {
entityTypesMigration4.run();
typeToEntityMigration5.run();
removeFalsePositiveManualRedactions6.run();
deleteRemovedManualAddRedactions7.run();
log.info("Migration is finished");
System.exit(SpringApplication.exit(ctx, () -> 0));

73
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DeleteRemovedManualAddRedactions7.java Normal file
View File

@ -0,0 +1,73 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.peristence.v1.server.service.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class DeleteRemovedManualAddRedactions7 extends Migration {
private static final String NAME = "Delete removed ManualAddRedactions";
private static final long VERSION = 7;
@Autowired
private AddRedactionPersistenceService addRedactionPersistenceService;
@Autowired
private RemoveRedactionPersistenceService removeRedactionPersistenceService;
@Autowired
private ManualRedactionProviderService manualRedactionProviderService;
public DeleteRemovedManualAddRedactions7() {
super(NAME, VERSION);
}
@Override
protected void migrate() {
List<ManualRedactionEntryEntity> addRedactions = addRedactionPersistenceService.findAllAddRedactions();
for (IdRemovalEntity idRemovalEntity : removeRedactionPersistenceService.findAllRemoveRedactions()) {
if (manualAddRedactionsContains(addRedactions, idRemovalEntity.getId().getAnnotationId())) {
log.info("hard delete ManualRedactions for file {} and annotation {}", idRemovalEntity.getId().getFileId(), idRemovalEntity.getId().getAnnotationId());
manualRedactionProviderService.hardDeleteManualRedactions(idRemovalEntity.getId().getFileId(), idRemovalEntity.getId().getAnnotationId());
}
}
}
private boolean manualAddRedactionsContains(List<ManualRedactionEntryEntity> addRedactions, String annotationId) {
for (ManualRedactionEntryEntity manualRedactionEntryEntity : addRedactions) {
if (StringUtils.equals(annotationId, manualRedactionEntryEntity.getId().getAnnotationId())) {
return true;
}
}
return false;
}
}

87
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionProviderService.java
View File

@ -1,23 +1,44 @@
package com.iqser.red.service.peristence.v1.server.service;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.utils.ManualImageRecategorizationMapper;
import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper;
import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.util.*;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class ManualRedactionProviderService {
@ -30,10 +51,10 @@ public class ManualRedactionProviderService {
private final LegalBasisChangePersistenceService legalBasisChangePersistenceService;
private final ResizeRedactionPersistenceService resizeRedactionPersistenceService;
@Transactional
public ManualRedactions getManualRedactions(String fileId) {
Set<ManualRedactionEntry> entriesToAdd = convertEntriesToAdd(addRedactionPersistenceService.findAddRedactions(fileId, false));
Set<IdRemoval> removals = convert(removeRedactionPersistenceService.findRemoveRedactions(fileId, false), IdRemoval.class);
@ -53,17 +74,53 @@ public class ManualRedactionProviderService {
comments.put(s, convert(c, Comment.class));
});
return new ManualRedactions(removals, entriesToAdd, forceRedactions, recategorizations, legalBasisChanges, resizeRedactions, comments);
}
public void hardDeleteManualRedactions(String fileId, String annotationId) {
try {
addRedactionPersistenceService.hardDelete(fileId, annotationId);
} catch (EmptyResultDataAccessException e) {
log.info("Ignored silently during deletion: {}", e.getMessage());
}
try {
removeRedactionPersistenceService.hardDelete(fileId, annotationId);
} catch (EmptyResultDataAccessException e) {
log.info("Ignored silently during deletion: {}", e.getMessage());
}
try {
forceRedactionPersistenceService.hardDelete(fileId, annotationId);
} catch (EmptyResultDataAccessException e) {
log.info("Ignored silently during deletion: {}", e.getMessage());
}
try {
recategorizationPersistenceService.hardDelete(fileId, annotationId);
} catch (EmptyResultDataAccessException e) {
log.info("Ignored silently during deletion: {}", e.getMessage());
}
try {
legalBasisChangePersistenceService.hardDelete(fileId, annotationId);
} catch (EmptyResultDataAccessException e) {
log.info("Ignored silently during deletion: {}", e.getMessage());
}
try {
resizeRedactionPersistenceService.hardDelete(fileId, annotationId);
} catch (EmptyResultDataAccessException e) {
log.info("Ignored silently during deletion: {}", e.getMessage());
}
try {
commentPersistenceService.hardDelete(fileId, annotationId);
} catch (EmptyResultDataAccessException e) {
log.info("Ignored silently during deletion: {}", e.getMessage());
}
}
private Set<ManualRedactionEntry> convertEntriesToAdd(Set<ManualRedactionEntryEntity> source) {
return source.stream().map(entry ->
convert(entry, ManualRedactionEntry.class, new ManualRedactionMapper())
).collect(Collectors.toSet());
return source.stream().map(entry -> convert(entry, ManualRedactionEntry.class, new ManualRedactionMapper())).collect(Collectors.toSet());
}
}

65
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -15,7 +15,6 @@ import java.util.stream.Collectors;
import javax.transaction.Transactional;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
@ -50,6 +49,16 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.CommentRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -153,37 +162,43 @@ public class ManualRedactionService {
.ifPresent((p) -> new BadRequestException("Invalid page found in the request"));
}
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId,
List<RemoveRedactionRequest> removeRedactionRequests) {
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId, List<RemoveRedactionRequest> removeRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId);
var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId);
for (var removeRedactionRequest : removeRedactionRequests) {
removeRedactionPersistenceService.insert(fileId, removeRedactionRequest);
if (manualAddRedactionsContains(manualRedactions, removeRedactionRequest.getAnnotationId()) && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) {
log.info("hard delete ManualRedactions for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
manualRedactionProviderService.hardDeleteManualRedactions(fileId, removeRedactionRequest.getAnnotationId());
actionPerformed = true;
} else {
Long commentId = null;
if (removeRedactionRequest.getComment() != null) {
commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId();
log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
removeRedactionPersistenceService.insert(fileId, removeRedactionRequest);
Long commentId = null;
if (removeRedactionRequest.getComment() != null) {
commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId();
}
if (!removeRedactionRequest.isRemoveFromDictionary()) {
Optional<RedactionLogEntry> redactionLogEntryOptional = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(removeRedactionRequest.getAnnotationId()))
.findFirst();
actionPerformed = actionPerformed || redactionLogEntryOptional.isPresent() && redactionLogEntryOptional.get().isHint();
}
actionPerformed = actionPerformed || handleRemoveFromDictionary(redactionLog, dossier, fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), removeRedactionRequest.isRemoveFromDictionary(), false);
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
if (!removeRedactionRequest.isRemoveFromDictionary()) {
Optional<RedactionLogEntry> redactionLogEntryOptional = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(removeRedactionRequest.getAnnotationId()))
.findFirst();
actionPerformed = actionPerformed || redactionLogEntryOptional.isPresent() && redactionLogEntryOptional.get().isHint();
}
actionPerformed = actionPerformed || handleRemoveFromDictionary(redactionLog, dossier, fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), removeRedactionRequest.isRemoveFromDictionary(), false);
response.add(ManualAddResponse.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.commentId(commentId)
.build());
}
if (actionPerformed) {
@ -806,4 +821,10 @@ public class ManualRedactionService {
}
}
private boolean manualAddRedactionsContains(ManualRedactions manualRedactions, String annotationId) {
return manualRedactions.getEntriesToAdd().stream().anyMatch(m -> annotationId.equals(m.getAnnotationId()));
}
}