From 02fd8aaf4978abaa1d0187b3fca13db9de00a43b Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 9 Mar 2022 19:17:42 +0200 Subject: [PATCH 1/2] bulk add --- .../resources/ManualRedactionResource.java | 4 +- .../controller/ManualRedactionController.java | 6 +- .../service/ManualRedactionService.java | 124 ++++++++---------- 3 files changed, 60 insertions(+), 74 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java index f6b8f3bd5..f2c904699 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java @@ -29,8 +29,8 @@ public interface ManualRedactionResource { @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - ManualAddResponse addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, - @RequestBody AddRedactionRequest addRedactionRequest); + List addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody List addRedactionRequests); @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java index 7ed73ea1c..92adf9c64 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java @@ -24,11 +24,11 @@ public class ManualRedactionController implements ManualRedactionResource { @Override - public ManualAddResponse addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, + public List addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, - @RequestBody AddRedactionRequest addRedactionRequest) { + @RequestBody List addRedactionRequests) { - return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequest); + return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index 471c94896..e970050eb 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -1,22 +1,5 @@ package com.iqser.red.service.peristence.v1.server.service; -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; -import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; - -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import javax.transaction.Transactional; - -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.hash.HashFunction; @@ -25,14 +8,7 @@ import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfigu import com.iqser.red.service.peristence.v1.server.controller.DictionaryController; import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper; import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -40,23 +16,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.CommentRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualAddResponse; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -65,10 +26,21 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.MessageType; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - import feign.FeignException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; +import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; @Slf4j @Service @@ -97,45 +69,59 @@ public class ManualRedactionService { private final HashFunction hashFunction = Hashing.murmur3_128(); - public ManualAddResponse addAddRedaction(String dossierId, String fileId, AddRedactionRequest addRedactionRequest) { + public List addAddRedaction(String dossierId, String fileId, List addRedactionRequests) { + var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); + var actionPerformed = false; - if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) { - try { - if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) { - throw new ConflictException("The entry you are trying to add is a stopword"); - } - dictionaryController.getDictionaryForType(addRedactionRequest.getTypeId()); - } catch (NotFoundException e) { - throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId()); + for (var addRedactionRequest : addRedactionRequests) { + if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) { + validateDictionary(addRedactionRequest); } + + String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); + addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); + + Long commentId = null; + if (addRedactionRequest.getComment() != null) { + commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId(); + } + + actionPerformed = actionPerformed || handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType()); + + response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } - String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); - addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); - - Long commentId = null; - if (addRedactionRequest.getComment() != null) { - commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId(); - } - - var actionPerformed = handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType()); - if (actionPerformed) { // in case of add to dict, there is no need to process surrounding text reprocess(dossierId, fileId); - } else { - if (!addRedactionRequest.isAddToDictionary() && !addRedactionRequest.isAddToDossierDictionary() && !addRedactionRequest.isRectangle()) { - var loaded = convert(getAddRedaction(fileId, annotationId), ManualRedactionEntry.class, new ManualRedactionMapper()); - ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(Set.of(loaded)).build(); - addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - } + } + + var manualTextRedactions = convert(response.stream() + .map(r -> getAddRedaction(fileId, r.getAnnotationId())) + .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) + .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); + + if (!manualTextRedactions.isEmpty()) { + ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); + addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build(); + return response; + } + + private void validateDictionary(AddRedactionRequest addRedactionRequest) { + try { + if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) { + throw new ConflictException("The entry you are trying to add is a stopword"); + } + dictionaryController.getDictionaryForType(addRedactionRequest.getTypeId()); + } catch (NotFoundException e) { + throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId()); + } } From c7dc76d123b3268593df4d4b59073fd5de9cafd8 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 9 Mar 2022 20:00:03 +0200 Subject: [PATCH 2/2] test fix --- .../peristence/v1/server/integration/tests/FileTest.java | 9 +++++---- .../v1/server/integration/tests/ManualRedactionTest.java | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index cbe6763bc..b9d7e0188 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -258,7 +258,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); - var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, AddRedactionRequest.builder() + var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, Collections.singletonList(AddRedactionRequest.builder() .addToDictionary(true) .addToDossierDictionary(false) .comment("comment") @@ -268,7 +268,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { .reason("1") .value("test") .legalBasis("1") - .build()); + .build())).iterator().next(); var removeRedaction = manualRedactionClient.addRemoveRedaction(dossierId, fileId, List.of(RemoveRedactionRequest.builder() .annotationId(addRedaction.getAnnotationId()) .comment("comment") @@ -343,8 +343,9 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); - var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, AddRedactionRequest.builder().addToDictionary(true) - .addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId).user("user").reason("1").value("test").legalBasis("1").build()); + var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, Collections.singletonList(AddRedactionRequest.builder().addToDictionary(true) + .addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId) + .user("user").reason("1").value("test").legalBasis("1").build())).iterator().next(); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 1a3bcbbfa..4ee2fe167 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -2,6 +2,7 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -56,7 +57,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var loadedComment = manualRedactionClient.getComment(comment.getId()); assertThat(loadedComment.getText()).isEqualTo("test"); - var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), AddRedactionRequest.builder() + var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder() .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build())) .section("section test") .addToDictionary(true) @@ -70,7 +71,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .rectangle(true) .textAfter("Text After") .textBefore("Text Before") - .build()); + .build())).iterator().next(); var loadedAddRedaction = manualRedactionClient.getAddRedaction(file.getId(), addRedaction.getAnnotationId()); assertThat(loadedAddRedaction.getUser()).isEqualTo("user"); assertThat(loadedAddRedaction.getType()).contains("manual"); @@ -93,7 +94,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { loadedAddRedaction = manualRedactionClient.getAddRedaction(file.getId(), addRedaction.getAnnotationId()); assertThat(loadedAddRedaction.getStatus()).isEqualTo(AnnotationStatus.DECLINED); - var addRedaction2 = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), AddRedactionRequest.builder() + var addRedaction2 = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder() .addToDictionary(true) .addToDossierDictionary(false) .status(AnnotationStatus.APPROVED) @@ -104,7 +105,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .legalBasis("1") .comment("comment") .section("section2") - .build()); + .build())).iterator().next(); manualRedactionClient.updateAddRedactionStatus(dossier.getId(), file.getId(), UpdateRedactionRequest.builder()