Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

bulk add

Browse Source
This commit is contained in:
Timo Bejan 2022-03-09 19:17:42 +02:00
parent daf19b4b93
commit 02fd8aaf49
3 changed files with 60 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java
View File

@ -29,8 +29,8 @@ public interface ManualRedactionResource {
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
ManualAddResponse addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId,
@RequestBody AddRedactionRequest addRedactionRequest);
List<ManualAddResponse> addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId,
@RequestBody List<AddRedactionRequest> addRedactionRequests);
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java
View File

@ -24,11 +24,11 @@ public class ManualRedactionController implements ManualRedactionResource {
@Override
public ManualAddResponse addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId,
public List<ManualAddResponse> addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody AddRedactionRequest addRedactionRequest) {
@RequestBody List<AddRedactionRequest> addRedactionRequests) {
return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequest);
return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests);
}

124
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -1,22 +1,5 @@
package com.iqser.red.service.peristence.v1.server.service;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import javax.transaction.Transactional;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.hash.HashFunction;
@ -25,14 +8,7 @@ import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfigu
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper;
import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@ -40,23 +16,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.CommentRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -65,10 +26,21 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.MessageType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
@Slf4j
@Service
@ -97,45 +69,59 @@ public class ManualRedactionService {
private final HashFunction hashFunction = Hashing.murmur3_128();
public ManualAddResponse addAddRedaction(String dossierId, String fileId, AddRedactionRequest addRedactionRequest) {
public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, List<AddRedactionRequest> addRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) {
try {
if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) {
throw new ConflictException("The entry you are trying to add is a stopword");
}
dictionaryController.getDictionaryForType(addRedactionRequest.getTypeId());
} catch (NotFoundException e) {
throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId());
for (var addRedactionRequest : addRedactionRequests) {
if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) {
validateDictionary(addRedactionRequest);
}
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
Long commentId = null;
if (addRedactionRequest.getComment() != null) {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
}
actionPerformed = actionPerformed || handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType());
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
}
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
Long commentId = null;
if (addRedactionRequest.getComment() != null) {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
}
var actionPerformed = handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType());
if (actionPerformed) {
// in case of add to dict, there is no need to process surrounding text
reprocess(dossierId, fileId);
} else {
if (!addRedactionRequest.isAddToDictionary() && !addRedactionRequest.isAddToDossierDictionary() && !addRedactionRequest.isRectangle()) {
var loaded = convert(getAddRedaction(fileId, annotationId), ManualRedactionEntry.class, new ManualRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(Set.of(loaded)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
}
var manualTextRedactions = convert(response.stream()
.map(r -> getAddRedaction(fileId, r.getAnnotationId()))
.filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle())
.collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
if (!manualTextRedactions.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build();
return response;
}
private void validateDictionary(AddRedactionRequest addRedactionRequest) {
try {
if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) {
throw new ConflictException("The entry you are trying to add is a stopword");
}
dictionaryController.getDictionaryForType(addRedactionRequest.getTypeId());
} catch (NotFoundException e) {
throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId());
}
}