diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java index dd2837135..fe4e07c5e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java @@ -12,7 +12,6 @@ import org.springframework.amqp.rabbit.annotation.RabbitHandler; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.core.context.SecurityContextImpl; import org.springframework.stereotype.Service; @@ -27,6 +26,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService; import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory; import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FoundTerm; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; @@ -34,6 +35,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; import lombok.AccessLevel; @@ -56,6 +58,7 @@ public class SearchTermOccurrencesResponseReceiver { AuditPersistenceService auditPersistenceService; FileStatusPersistenceService fileStatusPersistenceService; WebsocketService webSocketService; + EntityLogMongoService entityLogMongoService; @RabbitHandler @@ -85,6 +88,14 @@ public class SearchTermOccurrencesResponseReceiver { var dossier = dossierManagementService.getDossierById(response.getDossierId(), false, false); + var foundValue = response.getFoundTerms() + .get(0).value(); + List> positionsList = response.getFoundTerms() + .stream() + .map(FoundTerm::positions) + .collect(Collectors.toList()); + var entityLogEntries = entityLogMongoService.findDictionaryEntityLogEntriesByValueAndPositions(response.getDossierId(), response.getFileId(), foundValue, positionsList); + Set addRedactionRequests = response.getFoundTerms() .stream() .map(term -> AddRedactionRequestModel.builder() @@ -95,6 +106,7 @@ public class SearchTermOccurrencesResponseReceiver { .positions(convertPositions(term.positions())) .section(response.getSection()) .comment(response.getComment() == null ? null : new AddCommentRequestModel(response.getComment())) + .sourceId(getAnnotationId(entityLogEntries, term)) .build()) .collect(Collectors.toSet()); @@ -125,6 +137,18 @@ public class SearchTermOccurrencesResponseReceiver { } + private String getAnnotationId(Set entityLogEntries, FoundTerm term) { + + var optionalEntry = entityLogEntries.stream() + .filter(entry -> entry.getPositions().equals(term.positions())) + .findFirst(); + if (optionalEntry.isPresent()) { + return optionalEntry.get().getId(); + } + return null; + } + + private List convertPositions(List positions) { return positions.stream() diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml index 8a00cca7f..5b65033db 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml @@ -4,9 +4,10 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd"> - - - - + + + + + diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/tenant/7-add-entity-log-value-index.xml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/tenant/7-add-entity-log-value-index.xml new file mode 100644 index 000000000..5f2045835 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/tenant/7-add-entity-log-value-index.xml @@ -0,0 +1,23 @@ + + + + + + + { + "entityLogId": 1, + "value": 1, + } + + + {name: "entityLogId_value_index"} + + + + + \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 094b1c43f..feb12234f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -4174,4 +4174,151 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { } } } + + @Test + public void testSingleForceAndBulkForceOnDictEntryAutomaticAnalysisOff() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), true); + whenGetEntityLogInvocation(); + + var type1 = typeProvider.testAndProvideType(dossierTemplate, dossier, "test1", false, 70); + + List entityLogEntries = new ArrayList<>(); + String legal2 = "Legal 2"; + String legal3 = "Legal 3"; + String darthVader = "Darth Vader"; + List position1 = List.of(new Position(56.8f, 528.9f, 120.96f, 12.64f, 1)); + List position2 = List.of(new Position(56.8f, 501.3f, 120.96f, 12.64f, 1)); + List position3 = List.of(new Position(56.8f, 473.7f, 120.96f, 12.64f, 1)); + List position4 = List.of(new Position(56.8f, 446.1f, 120.96f, 12.64f, 1)); + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId1") + .type(type1.getType()) + .value(darthVader) + .entryType(EntryType.ENTITY) + .state(EntryState.SKIPPED) + .section("section") + .legalBasis("Legal 1") + .engines(Set.of(Engine.DOSSIER_DICTIONARY)) + .dictionaryEntry(true) + .dossierDictionaryEntry(true) + .positions(position1) + .build()); + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId2") + .type(type1.getType()) + .value(darthVader) + .entryType(EntryType.ENTITY) + .state(EntryState.SKIPPED) + .section("section") + .legalBasis("Legal 1") + .engines(Set.of(Engine.DOSSIER_DICTIONARY)) + .dictionaryEntry(true) + .dossierDictionaryEntry(true) + .positions(position2) + .build()); + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId3") + .type(type1.getType()) + .value(darthVader) + .entryType(EntryType.ENTITY) + .state(EntryState.SKIPPED) + .section("section") + .legalBasis("Legal 1") + .engines(Set.of(Engine.DOSSIER_DICTIONARY)) + .dictionaryEntry(true) + .dossierDictionaryEntry(true) + .positions(position3) + .build()); + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId4") + .type(type1.getType()) + .value(darthVader) + .entryType(EntryType.ENTITY) + .state(EntryState.SKIPPED) + .section("section") + .legalBasis("Legal 1") + .engines(Set.of(Engine.DOSSIER_DICTIONARY)) + .dictionaryEntry(true) + .dossierDictionaryEntry(true) + .positions(position4) + .build()); + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + // single force + manualRedactionClient.forceRedactionBulk(dossier.getId(), file.getId(), Set.of(ForceRedactionRequestModel.builder() + .annotationId("AnnotationId1") + .legalBasis(legal2) + .build())); + + var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, false); + + assertEquals(allManualRedactions.getForceRedactions().size(), 1); + assertEquals(allManualRedactions.getEntriesToAdd().size(), 1); + var entryToAdd = allManualRedactions.getEntriesToAdd() + .stream() + .findFirst() + .get(); + assertFalse(entryToAdd.isAddToDictionary()); + assertFalse(entryToAdd.isAddToDossierDictionary()); + assertEquals(entryToAdd.getValue(), darthVader); + + //bulk force + manualRedactionClient.addRedactionBulkLocal(dossier.getId(), + file.getId(), + AddRedactionBulkLocalRequestModel.builder() + .type(type1.getType()) + .value(darthVader) + .legalBasis(legal3) + .reason("reason") + .section("section") + .positions(List.of(new Rectangle(56.8f, 528.9f, 120.96f, 12.64f, 0))) + .build()); + List foundTerms = new ArrayList<>(); + foundTerms.add(new FoundTerm(position1, darthVader)); + foundTerms.add(new FoundTerm(position2, darthVader)); + foundTerms.add(new FoundTerm(position3, darthVader)); + foundTerms.add(new FoundTerm(position4, darthVader)); + + BulkLocalResponse addValueBackResponse = BulkLocalResponse.builder() + .fileId(file.getId()) + .dossierId(dossier.getId()) + .type(type1.getType()) + .legalBasis(legal3) + .reason("reason") + .section("section") + .foundTerms(foundTerms) + .userId("user") + .build(); + searchTermOccurrencesResponseReceiver.receive(addValueBackResponse); + + allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, false); + // check that there is a link to the dictionary entries. the merge of entity log will take case of them based on the sourceId + assertEquals(allManualRedactions.getForceRedactions().size(), 1); + assertEquals(allManualRedactions.getEntriesToAdd().size(), 5); + var addLocalBulk1 = allManualRedactions.getEntriesToAdd().stream().filter(e -> convertPositions(position1).equals(e.getPositions())).findFirst(); + assertThat(addLocalBulk1.isPresent()).isTrue(); + assertThat(addLocalBulk1.get().getSourceId()).isEqualTo("AnnotationId1"); + var addLocalBulk2 = allManualRedactions.getEntriesToAdd().stream().filter(e -> convertPositions(position2).equals(e.getPositions())).findFirst(); + assertThat(addLocalBulk2.isPresent()).isTrue(); + assertThat(addLocalBulk2.get().getSourceId()).isEqualTo("AnnotationId2"); + var addLocalBulk3 = allManualRedactions.getEntriesToAdd().stream().filter(e -> convertPositions(position3).equals(e.getPositions())).findFirst(); + assertThat(addLocalBulk3.isPresent()).isTrue(); + assertThat(addLocalBulk3.get().getSourceId()).isEqualTo("AnnotationId3"); + var addLocalBulk4 = allManualRedactions.getEntriesToAdd().stream().filter(e -> convertPositions(position4).equals(e.getPositions())).findFirst(); + assertThat(addLocalBulk4.isPresent()).isTrue(); + assertThat(addLocalBulk4.get().getSourceId()).isEqualTo("AnnotationId4"); + } + + private List convertPositions(List positions) { + + return positions.stream() + .map(position -> Rectangle.builder().page(position.getPageNumber()).height(position.h()).width(position.w()).topLeftX(position.x()).topLeftY(position.y()).build()) + .toList(); + } } diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentRepository.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentRepository.java index 929b8d674..15c44109d 100644 --- a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentRepository.java +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentRepository.java @@ -8,6 +8,7 @@ import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.mongodb.repository.Query; import org.springframework.stereotype.Repository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.projections.EntryWithManualChangesProjection; @@ -91,4 +92,13 @@ public interface EntityLogEntryDocumentRepository extends MongoRepository findAppliedEntitiesWhereLegalBasisEmpty(); + @Query("{ 'entityLogId': ?0," + + " 'value': ?1, " + + " '$or': [ { 'dictionaryEntry': true }, { 'dossierDictionaryEntry': true } ], " + + " 'positions': { $in: ?2 } " + + "}") + List findDictionaryEntityLogEntriesByValueAndPositions(String entityLogId, + String value, + List> positionsList); + } diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java index 0d4e1d593..884c1f6ea 100644 --- a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogDocument; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.exception.DocumentNotFoundException; @@ -407,4 +408,17 @@ public class EntityLogMongoService { .collect(Collectors.toSet()); } + public Set findDictionaryEntityLogEntriesByValueAndPositions(String dossierId, + String fileId, + String value, + List> positionsList) { + + return new HashSet<>(entityLogEntryDocumentRepository.findDictionaryEntityLogEntriesByValueAndPositions(mapper.getLogId(dossierId, fileId), + value, + positionsList) + .stream() + .map(mapper::fromLogEntryDocument) + .toList()); + } + }