From 02fd8aaf4978abaa1d0187b3fca13db9de00a43b Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 9 Mar 2022 19:17:42 +0200 Subject: [PATCH] bulk add --- .../resources/ManualRedactionResource.java | 4 +- .../controller/ManualRedactionController.java | 6 +- .../service/ManualRedactionService.java | 124 ++++++++---------- 3 files changed, 60 insertions(+), 74 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java index f6b8f3bd5..f2c904699 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java @@ -29,8 +29,8 @@ public interface ManualRedactionResource { @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - ManualAddResponse addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, - @RequestBody AddRedactionRequest addRedactionRequest); + List addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody List addRedactionRequests); @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java index 7ed73ea1c..92adf9c64 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java @@ -24,11 +24,11 @@ public class ManualRedactionController implements ManualRedactionResource { @Override - public ManualAddResponse addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, + public List addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, - @RequestBody AddRedactionRequest addRedactionRequest) { + @RequestBody List addRedactionRequests) { - return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequest); + return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index 471c94896..e970050eb 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -1,22 +1,5 @@ package com.iqser.red.service.peristence.v1.server.service; -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; -import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; - -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import javax.transaction.Transactional; - -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.hash.HashFunction; @@ -25,14 +8,7 @@ import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfigu import com.iqser.red.service.peristence.v1.server.controller.DictionaryController; import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper; import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -40,23 +16,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.CommentRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualAddResponse; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -65,10 +26,21 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.MessageType; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - import feign.FeignException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; +import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; @Slf4j @Service @@ -97,45 +69,59 @@ public class ManualRedactionService { private final HashFunction hashFunction = Hashing.murmur3_128(); - public ManualAddResponse addAddRedaction(String dossierId, String fileId, AddRedactionRequest addRedactionRequest) { + public List addAddRedaction(String dossierId, String fileId, List addRedactionRequests) { + var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); + var actionPerformed = false; - if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) { - try { - if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) { - throw new ConflictException("The entry you are trying to add is a stopword"); - } - dictionaryController.getDictionaryForType(addRedactionRequest.getTypeId()); - } catch (NotFoundException e) { - throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId()); + for (var addRedactionRequest : addRedactionRequests) { + if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) { + validateDictionary(addRedactionRequest); } + + String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); + addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); + + Long commentId = null; + if (addRedactionRequest.getComment() != null) { + commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId(); + } + + actionPerformed = actionPerformed || handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType()); + + response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } - String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); - addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); - - Long commentId = null; - if (addRedactionRequest.getComment() != null) { - commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId(); - } - - var actionPerformed = handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType()); - if (actionPerformed) { // in case of add to dict, there is no need to process surrounding text reprocess(dossierId, fileId); - } else { - if (!addRedactionRequest.isAddToDictionary() && !addRedactionRequest.isAddToDossierDictionary() && !addRedactionRequest.isRectangle()) { - var loaded = convert(getAddRedaction(fileId, annotationId), ManualRedactionEntry.class, new ManualRedactionMapper()); - ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(Set.of(loaded)).build(); - addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - } + } + + var manualTextRedactions = convert(response.stream() + .map(r -> getAddRedaction(fileId, r.getAnnotationId())) + .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) + .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); + + if (!manualTextRedactions.isEmpty()) { + ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); + addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build(); + return response; + } + + private void validateDictionary(AddRedactionRequest addRedactionRequest) { + try { + if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) { + throw new ConflictException("The entry you are trying to add is a stopword"); + } + dictionaryController.getDictionaryForType(addRedactionRequest.getTypeId()); + } catch (NotFoundException e) { + throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId()); + } }