Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-10104-10178' into 'master'

Browse Source 
RED-10104 && RED-10178

Closes RED-10104

See merge request redactmanager/persistence-service!783
This commit is contained in:
Maverick Studer 2024-10-14 14:30:46 +02:00
commit fef66f92b2
8 changed files with 201 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java vendored
View File

@ -46,6 +46,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
@ -211,17 +212,40 @@ public class ManualRedactionController implements ManualRedactionResource {
var dossier = dossierManagementService.getDossierById(dossierId, false, false);
dictionaryPersistenceService.getType(TypeIdUtils.toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId()));
fileStatusService.setStatusBulkLocalRedactionsProcessing(fileId, addRedactionRequest);
if (!addRedactionRequest.isRectangle()) {
fileStatusService.setStatusBulkLocalRedactionsProcessing(fileId, addRedactionRequest);
EntityLogEntry entityLogEntry = pendingEntryFactory.buildAddRedactionBulkLocalEntry(addRedactionRequest);
return ManualRedactionResponse.builder()
.manualAnnotationResponses(List.of(ManualAnnotationResponse.builder()
.annotationId(manualRedactionService.getNewAnnotationId())
.entityLogEntry(entityLogEntry)
.build()))
.build();
EntityLogEntry entityLogEntry = pendingEntryFactory.buildAddRedactionBulkLocalEntry(addRedactionRequest);
return ManualRedactionResponse.builder()
.manualAnnotationResponses(List.of(ManualAnnotationResponse.builder()
.annotationId(manualRedactionService.getNewAnnotationId())
.entityLogEntry(entityLogEntry)
.build()))
.build();
} else {
Set<AddRedactionRequestModel> addRedactionRequestModels = addRedactionRequest.getPageNumbers()
.stream()
.map(page -> AddRedactionRequestModel.builder()
.type(addRedactionRequest.getType())
.value(addRedactionRequest.getValue())
.reason(addRedactionRequest.getReason())
.legalBasis(addRedactionRequest.getLegalBasis())
.positions(addRedactionRequest.getPositions()
.stream()
.map(rectangle -> Rectangle.createRectangle(rectangle.getTopLeftX(),
rectangle.getTopLeftY(),
rectangle.getWidth(),
rectangle.getHeight(),
page))
.toList())
.comment(addRedactionRequest.getComment() != null ? AddCommentRequestModel.builder().text(addRedactionRequest.getComment()).build() : null)
.section(addRedactionRequest.getSection())
.rectangle(true)
.build())
.collect(Collectors.toSet());
return addRedactionBulk(dossierId, fileId, addRedactionRequestModels);
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -73,6 +73,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddR
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.llm.service.LlmNerMessage;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
@ -1060,6 +1061,7 @@ public class FileStatusService {
.section(addRedactionBulkLocalRequestModel.getSection())
.pageNumbers(addRedactionBulkLocalRequestModel.getPageNumbers())
.comment(addRedactionBulkLocalRequestModel.getComment())
.userId(KeycloakSecurity.getUserId())
.build();
addSearchTermOccurrencesAnalysisRequestToAnalysisQueue(fileStatus, bulkLocalRequest);
}

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java
View File

@ -10,6 +10,11 @@ import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContext;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.context.SecurityContextImpl;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.type.TypeReference;
@ -73,6 +78,11 @@ public class SearchTermOccurrencesResponseReceiver {
public void receive(BulkLocalResponse response) {
Authentication authentication = new UsernamePasswordAuthenticationToken(response.getUserId(), null, null);
SecurityContextImpl securityContext = new SecurityContextImpl();
securityContext.setAuthentication(authentication);
SecurityContextHolder.setContext(securityContext);
var dossier = dossierManagementService.getDossierById(response.getDossierId(), false, false);
Set<AddRedactionRequestModel> addRedactionRequests = response.getFoundTerms()

214
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -81,6 +81,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary.
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionBulkLocalRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
@ -3292,6 +3293,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
testRectangleRedactionsBulkLocal(dossierTemplate, dossier, file);
}
private void testBulkLocal(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) {
whenGetEntityLogInvocation();
@ -3342,25 +3344,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
String newLegal = "new Legal";
String otherSection = "other section";
String comment1 = "Recategorizing Luke Skywalker37";
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(
dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value("Luke Skywalker37")
.comment(comment1)
.build(),
false);
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value("Luke Skywalker37")
.comment(comment1)
.build(),
false);
assertEquals(1, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType());
assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getSection());
assertEquals(newType.getType(),
manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType());
assertEquals(newLegal,
manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection,
manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getSection());
Long commentId1 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId1);
@ -3369,26 +3373,28 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertEquals(comment1, commentEntity1.get().getText());
String comment2 = "Recategorizing Darth Vader with Legal 3";
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(
dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value(darthVader)
.originLegalBases(Set.of(legal3))
.comment(comment2)
.build(),
false);
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value(darthVader)
.originLegalBases(Set.of(legal3))
.comment(comment2)
.build(),
false);
assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses()
.get(1).getEntityLogEntry().getType());
assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses()
.get(29).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses()
.get(79).getEntityLogEntry().getSection());
assertEquals(newType.getType(),
manualRedactionResponse.getManualAnnotationResponses()
.get(1).getEntityLogEntry().getType());
assertEquals(newLegal,
manualRedactionResponse.getManualAnnotationResponses()
.get(29).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection,
manualRedactionResponse.getManualAnnotationResponses()
.get(79).getEntityLogEntry().getSection());
Long commentId2 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId2);
@ -3397,26 +3403,28 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertEquals(comment2, commentEntity2.get().getText());
String comment3 = "Recategorizing Darth Vader with type2";
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(
dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value(darthVader)
.originTypes(Set.of(type2.getType()))
.comment(comment3)
.build(),
false);
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value(darthVader)
.originTypes(Set.of(type2.getType()))
.comment(comment3)
.build(),
false);
assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType());
assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses()
.get(36).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses()
.get(98).getEntityLogEntry().getSection());
assertEquals(newType.getType(),
manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType());
assertEquals(newLegal,
manualRedactionResponse.getManualAnnotationResponses()
.get(36).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection,
manualRedactionResponse.getManualAnnotationResponses()
.get(98).getEntityLogEntry().getSection());
Long commentId3 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId3);
@ -3425,15 +3433,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertEquals(comment3, commentEntity3.get().getText());
String comment4 = "Removing all Darth Vader annotations";
manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(
dossier.getId(),
file.getId(),
RemoveRedactionBulkLocalRequestModel.builder()
.rectangle(false)
.value(darthVader)
.comment(comment4)
.build(),
false);
manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(),
file.getId(),
RemoveRedactionBulkLocalRequestModel.builder()
.rectangle(false)
.value(darthVader)
.comment(comment4)
.build(),
false);
assertEquals(202, manualRedactionResponse.getManualAnnotationResponses().size());
Long commentId4 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
@ -3444,7 +3451,6 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
}
private void testRectangleRedactionsBulkLocal(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) {
whenGetEntityLogInvocation();
@ -3768,6 +3774,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertEquals(EntryState.APPLIED, rectangleAnnotationEntry.getState());
String awesomeComment = "awesome comment";
String myUser = "MyUser";
BulkLocalResponse addValueBackResponse = BulkLocalResponse.builder()
.fileId(file.getId())
.dossierId(dossier.getId())
@ -3777,6 +3784,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section(section)
.foundTerms(List.of(new FoundTerm(List.of(position), value)))
.comment(awesomeComment)
.userId(myUser)
.build();
searchTermOccurrencesResponseReceiver.receive(addValueBackResponse);
@ -3801,6 +3809,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
List<CommentEntity> commentEntities = commentRepository.findByFileIdAndAnnotationId(file.getId(), newValueAnnotation.getId(), false);
assertEquals(commentEntities.size(), 1);
assertEquals(commentEntities.get(0).getText(), awesomeComment);
assertEquals(commentEntities.get(0).getUser(), myUser);
removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest, true);
@ -3843,6 +3852,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
commentEntities = commentRepository.findByFileIdAndAnnotationId(file.getId(), appliedValueAnnotations.get(0).getId(), false);
assertEquals(commentEntities.size(), 1);
assertEquals(commentEntities.get(0).getText(), awesomeComment);
assertEquals(commentEntities.get(0).getUser(), myUser);
String totallyDifferentComment = "totally different comment";
RemoveRedactionBulkLocalRequestModel removeRectangleRequest = RemoveRedactionBulkLocalRequestModel.builder()
@ -3924,6 +3934,78 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
}
@Test
public void testLocalAddWithRectangle() {
whenGetEntityLogInvocation();
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
var type = typeProvider.testAndProvideType(dossierTemplate, dossier, "test", false, 100);
var entityLog = new EntityLog(1, 1, new ArrayList<>(), null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
AddRedactionBulkLocalRequestModel addRedactionRequest = AddRedactionBulkLocalRequestModel.builder()
.type(type.getType())
.value("non-readable content")
.legalBasis("legal basis")
.reason("reason")
.section("section")
.comment("test comment")
.positions(List.of(new Rectangle(100f, 200f, 50f, 50f, 0)))
.pageNumbers(Set.of(1, 2, 3))
.rectangle(true)
.build();
ManualRedactionResponse response = manualRedactionClient.addRedactionBulkLocal(dossier.getId(), file.getId(), addRedactionRequest);
assertNotNull(response);
assertEquals(3, response.getManualAnnotationResponses().size());
EntityLog currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId());
assertNotNull(currentEntityLog);
assertEquals(3, currentEntityLog.getEntityLogEntry().size());
for (int i = 1; i <= 3; i++) {
int page = i;
EntityLogEntry entry = currentEntityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getPositions() != null
&& !e.getPositions().isEmpty()
&& e.getPositions()
.get(0).getPageNumber() == page)
.findFirst()
.orElse(null);
assertNotNull(entry);
assertEquals("non-readable content", entry.getValue());
assertEquals("legal basis", entry.getLegalBasis());
assertEquals("reason", entry.getReason());
assertEquals("section", entry.getSection());
assertEquals(type.getType(), entry.getType());
assertEquals(EntryType.AREA, entry.getEntryType());
assertEquals(EntryState.APPLIED, entry.getState());
assertEquals(1, entry.getPositions().size());
Position position = entry.getPositions()
.get(0);
assertEquals(page, position.getPageNumber());
assertEquals(100f, position.x());
assertEquals(200f, position.y());
assertEquals(50f, position.w());
assertEquals(50f, position.h());
}
for (EntityLogEntry entry : currentEntityLog.getEntityLogEntry()) {
List<CommentEntity> comments = commentRepository.findByFileIdAndAnnotationId(file.getId(), entry.getId(), false);
assertEquals(1, comments.size());
CommentEntity comment = comments.get(0);
assertEquals("test comment", comment.getText());
}
}
private Rectangle toRectangle(Position position) {
return new Rectangle(position.x(), position.y(), position.w(), position.h(), 0);

1
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java
View File

@ -69,6 +69,7 @@ public class SearchTermOccurrencesTest extends AbstractPersistenceServerServiceT
.section("section")
.foundTerms(List.of(new FoundTerm(List.of(new Position(new float[]{1f, 2f, 3f, 4f}, 1)), "searchTerm")))
.comment(null)
.userId("MyUser")
.build());
List<ManualRedactionEntryEntity> newEntries = manualRedactionRepository.findByFileIdAndOptions(file.getId(), false, false, false);

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java
View File

@ -36,4 +36,7 @@ public class BulkLocalRequest {
private String comment;
@NonNull
private String userId;
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java
View File

@ -39,4 +39,6 @@ public class BulkLocalResponse {
private String comment;
@NonNull
private String userId;
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java
View File

@ -35,6 +35,8 @@ public class AddRedactionBulkLocalRequestModel {
private String section;
private boolean rectangle;
@NonNull
@Builder.Default
private List<Rectangle> positions = new ArrayList<>();