Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #280: bulk add

Browse Source 
Merge in RED/persistence-service from bulk-add-action to master

* commit 'c7dc76d123b3268593df4d4b59073fd5de9cafd8':
  test fix
  bulk add
This commit is contained in:
Timo Bejan 2022-03-09 19:03:17 +01:00
commit b2082ed2c5
5 changed files with 70 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ManualRedactionResource.java
View File

@ -29,8 +29,8 @@ public interface ManualRedactionResource {
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
ManualAddResponse addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId,
@RequestBody AddRedactionRequest addRedactionRequest);
List<ManualAddResponse> addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId,
@RequestBody List<AddRedactionRequest> addRedactionRequests);
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java
View File

@ -24,11 +24,11 @@ public class ManualRedactionController implements ManualRedactionResource {
@Override
public ManualAddResponse addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId,
public List<ManualAddResponse> addAddRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody AddRedactionRequest addRedactionRequest) {
@RequestBody List<AddRedactionRequest> addRedactionRequests) {
return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequest);
return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests);
}

124
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -1,22 +1,5 @@
package com.iqser.red.service.peristence.v1.server.service;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import javax.transaction.Transactional;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.hash.HashFunction;
@ -25,14 +8,7 @@ import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfigu
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper;
import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@ -40,23 +16,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.CommentRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -65,10 +26,21 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.MessageType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
@Slf4j
@Service
@ -97,45 +69,59 @@ public class ManualRedactionService {
private final HashFunction hashFunction = Hashing.murmur3_128();
public ManualAddResponse addAddRedaction(String dossierId, String fileId, AddRedactionRequest addRedactionRequest) {
public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, List<AddRedactionRequest> addRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) {
try {
if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) {
throw new ConflictException("The entry you are trying to add is a stopword");
}
dictionaryController.getDictionaryForType(addRedactionRequest.getTypeId());
} catch (NotFoundException e) {
throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId());
for (var addRedactionRequest : addRedactionRequests) {
if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) {
validateDictionary(addRedactionRequest);
}
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
Long commentId = null;
if (addRedactionRequest.getComment() != null) {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
}
actionPerformed = actionPerformed || handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType());
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
}
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
Long commentId = null;
if (addRedactionRequest.getComment() != null) {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
}
var actionPerformed = handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType());
if (actionPerformed) {
// in case of add to dict, there is no need to process surrounding text
reprocess(dossierId, fileId);
} else {
if (!addRedactionRequest.isAddToDictionary() && !addRedactionRequest.isAddToDossierDictionary() && !addRedactionRequest.isRectangle()) {
var loaded = convert(getAddRedaction(fileId, annotationId), ManualRedactionEntry.class, new ManualRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(Set.of(loaded)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
}
var manualTextRedactions = convert(response.stream()
.map(r -> getAddRedaction(fileId, r.getAnnotationId()))
.filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle())
.collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
if (!manualTextRedactions.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build();
return response;
}
private void validateDictionary(AddRedactionRequest addRedactionRequest) {
try {
if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) {
throw new ConflictException("The entry you are trying to add is a stopword");
}
dictionaryController.getDictionaryForType(addRedactionRequest.getTypeId());
} catch (NotFoundException e) {
throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId());
}
}

9
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -258,7 +258,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, AddRedactionRequest.builder()
var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, Collections.singletonList(AddRedactionRequest.builder()
.addToDictionary(true)
.addToDossierDictionary(false)
.comment("comment")
@ -268,7 +268,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
.reason("1")
.value("test")
.legalBasis("1")
.build());
.build())).iterator().next();
var removeRedaction = manualRedactionClient.addRemoveRedaction(dossierId, fileId, List.of(RemoveRedactionRequest.builder()
.annotationId(addRedaction.getAnnotationId())
.comment("comment")
@ -343,8 +343,9 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, AddRedactionRequest.builder().addToDictionary(true)
.addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId).user("user").reason("1").value("test").legalBasis("1").build());
var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, Collections.singletonList(AddRedactionRequest.builder().addToDictionary(true)
.addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId)
.user("user").reason("1").value("test").legalBasis("1").build())).iterator().next();
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());

9
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@ -56,7 +57,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var loadedComment = manualRedactionClient.getComment(comment.getId());
assertThat(loadedComment.getText()).isEqualTo("test");
var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), AddRedactionRequest.builder()
var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder()
.positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build()))
.section("section test")
.addToDictionary(true)
@ -70,7 +71,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.rectangle(true)
.textAfter("Text After")
.textBefore("Text Before")
.build());
.build())).iterator().next();
var loadedAddRedaction = manualRedactionClient.getAddRedaction(file.getId(), addRedaction.getAnnotationId());
assertThat(loadedAddRedaction.getUser()).isEqualTo("user");
assertThat(loadedAddRedaction.getType()).contains("manual");
@ -93,7 +94,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
loadedAddRedaction = manualRedactionClient.getAddRedaction(file.getId(), addRedaction.getAnnotationId());
assertThat(loadedAddRedaction.getStatus()).isEqualTo(AnnotationStatus.DECLINED);
var addRedaction2 = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), AddRedactionRequest.builder()
var addRedaction2 = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder()
.addToDictionary(true)
.addToDossierDictionary(false)
.status(AnnotationStatus.APPROVED)
@ -104,7 +105,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.legalBasis("1")
.comment("comment")
.section("section2")
.build());
.build())).iterator().next();
manualRedactionClient.updateAddRedactionStatus(dossier.getId(), file.getId(),
UpdateRedactionRequest.builder()