Merge branch 'RED-10425-bp' into 'release/2.589.x'
RED-10425 - Annotation added twice when bulk-force while auto-analysis is disabled See merge request redactmanager/persistence-service!856
This commit is contained in:
Corina Olariu
commit
17453a889f
@ -12,7 +12,6 @@ import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.context.SecurityContext;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.security.core.context.SecurityContextImpl;
|
||||
import org.springframework.stereotype.Service;
|
||||
@ -27,6 +26,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FoundTerm;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
@ -34,6 +35,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
|
||||
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
@ -56,6 +58,7 @@ public class SearchTermOccurrencesResponseReceiver {
|
||||
AuditPersistenceService auditPersistenceService;
|
||||
FileStatusPersistenceService fileStatusPersistenceService;
|
||||
WebsocketService webSocketService;
|
||||
EntityLogMongoService entityLogMongoService;
|
||||
|
||||
|
||||
@RabbitHandler
|
||||
@ -85,6 +88,14 @@ public class SearchTermOccurrencesResponseReceiver {
|
||||
|
||||
var dossier = dossierManagementService.getDossierById(response.getDossierId(), false, false);
|
||||
|
||||
var foundValue = response.getFoundTerms()
|
||||
.get(0).value();
|
||||
List<List<Position>> positionsList = response.getFoundTerms()
|
||||
.stream()
|
||||
.map(FoundTerm::positions)
|
||||
.collect(Collectors.toList());
|
||||
var entityLogEntries = entityLogMongoService.findDictionaryEntityLogEntriesByValueAndPositions(response.getDossierId(), response.getFileId(), foundValue, positionsList);
|
||||
|
||||
Set<AddRedactionRequestModel> addRedactionRequests = response.getFoundTerms()
|
||||
.stream()
|
||||
.map(term -> AddRedactionRequestModel.builder()
|
||||
@ -95,6 +106,7 @@ public class SearchTermOccurrencesResponseReceiver {
|
||||
.positions(convertPositions(term.positions()))
|
||||
.section(response.getSection())
|
||||
.comment(response.getComment() == null ? null : new AddCommentRequestModel(response.getComment()))
|
||||
.sourceId(getAnnotationId(entityLogEntries, term))
|
||||
.build())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
@ -125,6 +137,18 @@ public class SearchTermOccurrencesResponseReceiver {
|
||||
}
|
||||
|
||||
|
||||
private String getAnnotationId(Set<EntityLogEntry> entityLogEntries, FoundTerm term) {
|
||||
|
||||
var optionalEntry = entityLogEntries.stream()
|
||||
.filter(entry -> entry.getPositions().equals(term.positions()))
|
||||
.findFirst();
|
||||
if (optionalEntry.isPresent()) {
|
||||
return optionalEntry.get().getId();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> convertPositions(List<Position> positions) {
|
||||
|
||||
return positions.stream()
|
||||
|
||||
@ -4,9 +4,10 @@
|
||||
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd">
|
||||
<include file="/mongo/changelog/tenant/1-initial-database.changelog.xml"/>
|
||||
<include file="/mongo/changelog/tenant/2-create-indices-for-entries.xml"/>
|
||||
<include file="/mongo/changelog/tenant/3-add-page-paragraph-idx.xml"/>
|
||||
<!--<include file="/mongo/changelog/tenant/4-create-component-entities.xml"/>-->
|
||||
<include file="/mongo/changelog/tenant/5-add-duplicate-text-ranges.xml"/>
|
||||
<include file="/mongo/changelog/tenant/6-rename-component-collections.xml"/>
|
||||
<!-- <include file="/mongo/changelog/tenant/3-add-page-paragraph-idx.xml"/>-->
|
||||
<!-- <include file="/mongo/changelog/tenant/4-create-component-entities.xml"/>-->
|
||||
<!-- <include file="/mongo/changelog/tenant/5-add-duplicate-text-ranges.xml"/>-->
|
||||
<!-- <include file="/mongo/changelog/tenant/6-rename-component-collections.xml"/>-->
|
||||
<include file="/mongo/changelog/tenant/7-add-entity-log-value-index.xml"/>
|
||||
<!-- THIS FILE IS NOT RUN IN THE CURRENT CONFIGURATION, PLEASE ADD CHANGES TO redaction-service -->
|
||||
</databaseChangeLog>
|
||||
|
||||
@ -0,0 +1,23 @@
|
||||
<databaseChangeLog
|
||||
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
|
||||
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
|
||||
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
|
||||
|
||||
<changeSet id="createIndexForEntityLogIdAndValue" author="corina">
|
||||
|
||||
<ext:createIndex collectionName="entity-log-entries">
|
||||
<ext:keys>
|
||||
{
|
||||
"entityLogId": 1,
|
||||
"value": 1,
|
||||
}
|
||||
</ext:keys>
|
||||
<ext:options>
|
||||
{name: "entityLogId_value_index"}
|
||||
</ext:options>
|
||||
</ext:createIndex>
|
||||
|
||||
</changeSet>
|
||||
</databaseChangeLog>
|
||||
@ -4174,4 +4174,151 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleForceAndBulkForceOnDictEntryAutomaticAnalysisOff() {
|
||||
|
||||
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
|
||||
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
|
||||
var file = fileTesterAndProvider.testAndProvideFile(dossier);
|
||||
|
||||
fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), true);
|
||||
whenGetEntityLogInvocation();
|
||||
|
||||
var type1 = typeProvider.testAndProvideType(dossierTemplate, dossier, "test1", false, 70);
|
||||
|
||||
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
|
||||
String legal2 = "Legal 2";
|
||||
String legal3 = "Legal 3";
|
||||
String darthVader = "Darth Vader";
|
||||
List<Position> position1 = List.of(new Position(56.8f, 528.9f, 120.96f, 12.64f, 1));
|
||||
List<Position> position2 = List.of(new Position(56.8f, 501.3f, 120.96f, 12.64f, 1));
|
||||
List<Position> position3 = List.of(new Position(56.8f, 473.7f, 120.96f, 12.64f, 1));
|
||||
List<Position> position4 = List.of(new Position(56.8f, 446.1f, 120.96f, 12.64f, 1));
|
||||
entityLogEntries.add(EntityLogEntry.builder()
|
||||
.id("AnnotationId1")
|
||||
.type(type1.getType())
|
||||
.value(darthVader)
|
||||
.entryType(EntryType.ENTITY)
|
||||
.state(EntryState.SKIPPED)
|
||||
.section("section")
|
||||
.legalBasis("Legal 1")
|
||||
.engines(Set.of(Engine.DOSSIER_DICTIONARY))
|
||||
.dictionaryEntry(true)
|
||||
.dossierDictionaryEntry(true)
|
||||
.positions(position1)
|
||||
.build());
|
||||
entityLogEntries.add(EntityLogEntry.builder()
|
||||
.id("AnnotationId2")
|
||||
.type(type1.getType())
|
||||
.value(darthVader)
|
||||
.entryType(EntryType.ENTITY)
|
||||
.state(EntryState.SKIPPED)
|
||||
.section("section")
|
||||
.legalBasis("Legal 1")
|
||||
.engines(Set.of(Engine.DOSSIER_DICTIONARY))
|
||||
.dictionaryEntry(true)
|
||||
.dossierDictionaryEntry(true)
|
||||
.positions(position2)
|
||||
.build());
|
||||
entityLogEntries.add(EntityLogEntry.builder()
|
||||
.id("AnnotationId3")
|
||||
.type(type1.getType())
|
||||
.value(darthVader)
|
||||
.entryType(EntryType.ENTITY)
|
||||
.state(EntryState.SKIPPED)
|
||||
.section("section")
|
||||
.legalBasis("Legal 1")
|
||||
.engines(Set.of(Engine.DOSSIER_DICTIONARY))
|
||||
.dictionaryEntry(true)
|
||||
.dossierDictionaryEntry(true)
|
||||
.positions(position3)
|
||||
.build());
|
||||
entityLogEntries.add(EntityLogEntry.builder()
|
||||
.id("AnnotationId4")
|
||||
.type(type1.getType())
|
||||
.value(darthVader)
|
||||
.entryType(EntryType.ENTITY)
|
||||
.state(EntryState.SKIPPED)
|
||||
.section("section")
|
||||
.legalBasis("Legal 1")
|
||||
.engines(Set.of(Engine.DOSSIER_DICTIONARY))
|
||||
.dictionaryEntry(true)
|
||||
.dossierDictionaryEntry(true)
|
||||
.positions(position4)
|
||||
.build());
|
||||
var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
|
||||
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
|
||||
|
||||
// single force
|
||||
manualRedactionClient.forceRedactionBulk(dossier.getId(), file.getId(), Set.of(ForceRedactionRequestModel.builder()
|
||||
.annotationId("AnnotationId1")
|
||||
.legalBasis(legal2)
|
||||
.build()));
|
||||
|
||||
var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, false);
|
||||
|
||||
assertEquals(allManualRedactions.getForceRedactions().size(), 1);
|
||||
assertEquals(allManualRedactions.getEntriesToAdd().size(), 1);
|
||||
var entryToAdd = allManualRedactions.getEntriesToAdd()
|
||||
.stream()
|
||||
.findFirst()
|
||||
.get();
|
||||
assertFalse(entryToAdd.isAddToDictionary());
|
||||
assertFalse(entryToAdd.isAddToDossierDictionary());
|
||||
assertEquals(entryToAdd.getValue(), darthVader);
|
||||
|
||||
//bulk force
|
||||
manualRedactionClient.addRedactionBulkLocal(dossier.getId(),
|
||||
file.getId(),
|
||||
AddRedactionBulkLocalRequestModel.builder()
|
||||
.type(type1.getType())
|
||||
.value(darthVader)
|
||||
.legalBasis(legal3)
|
||||
.reason("reason")
|
||||
.section("section")
|
||||
.positions(List.of(new Rectangle(56.8f, 528.9f, 120.96f, 12.64f, 0)))
|
||||
.build());
|
||||
List<FoundTerm> foundTerms = new ArrayList<>();
|
||||
foundTerms.add(new FoundTerm(position1, darthVader));
|
||||
foundTerms.add(new FoundTerm(position2, darthVader));
|
||||
foundTerms.add(new FoundTerm(position3, darthVader));
|
||||
foundTerms.add(new FoundTerm(position4, darthVader));
|
||||
|
||||
BulkLocalResponse addValueBackResponse = BulkLocalResponse.builder()
|
||||
.fileId(file.getId())
|
||||
.dossierId(dossier.getId())
|
||||
.type(type1.getType())
|
||||
.legalBasis(legal3)
|
||||
.reason("reason")
|
||||
.section("section")
|
||||
.foundTerms(foundTerms)
|
||||
.userId("user")
|
||||
.build();
|
||||
searchTermOccurrencesResponseReceiver.receive(addValueBackResponse);
|
||||
|
||||
allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, false);
|
||||
// check that there is a link to the dictionary entries. the merge of entity log will take case of them based on the sourceId
|
||||
assertEquals(allManualRedactions.getForceRedactions().size(), 1);
|
||||
assertEquals(allManualRedactions.getEntriesToAdd().size(), 5);
|
||||
var addLocalBulk1 = allManualRedactions.getEntriesToAdd().stream().filter(e -> convertPositions(position1).equals(e.getPositions())).findFirst();
|
||||
assertThat(addLocalBulk1.isPresent()).isTrue();
|
||||
assertThat(addLocalBulk1.get().getSourceId()).isEqualTo("AnnotationId1");
|
||||
var addLocalBulk2 = allManualRedactions.getEntriesToAdd().stream().filter(e -> convertPositions(position2).equals(e.getPositions())).findFirst();
|
||||
assertThat(addLocalBulk2.isPresent()).isTrue();
|
||||
assertThat(addLocalBulk2.get().getSourceId()).isEqualTo("AnnotationId2");
|
||||
var addLocalBulk3 = allManualRedactions.getEntriesToAdd().stream().filter(e -> convertPositions(position3).equals(e.getPositions())).findFirst();
|
||||
assertThat(addLocalBulk3.isPresent()).isTrue();
|
||||
assertThat(addLocalBulk3.get().getSourceId()).isEqualTo("AnnotationId3");
|
||||
var addLocalBulk4 = allManualRedactions.getEntriesToAdd().stream().filter(e -> convertPositions(position4).equals(e.getPositions())).findFirst();
|
||||
assertThat(addLocalBulk4.isPresent()).isTrue();
|
||||
assertThat(addLocalBulk4.get().getSourceId()).isEqualTo("AnnotationId4");
|
||||
}
|
||||
|
||||
private List<Rectangle> convertPositions(List<Position> positions) {
|
||||
|
||||
return positions.stream()
|
||||
.map(position -> Rectangle.builder().page(position.getPageNumber()).height(position.h()).width(position.w()).topLeftX(position.x()).topLeftY(position.y()).build())
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ import org.springframework.data.mongodb.repository.MongoRepository;
|
||||
import org.springframework.data.mongodb.repository.Query;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.projections.EntryWithManualChangesProjection;
|
||||
|
||||
@ -91,4 +92,13 @@ public interface EntityLogEntryDocumentRepository extends MongoRepository<Entity
|
||||
""", fields = "{ 'entryId': 1, 'entityLogId': 1, 'state': 1, 'entryType': 1 , 'manualChanges': 1}")
|
||||
List<EntryWithManualChangesProjection> findAppliedEntitiesWhereLegalBasisEmpty();
|
||||
|
||||
@Query("{ 'entityLogId': ?0," +
|
||||
" 'value': ?1, " +
|
||||
" '$or': [ { 'dictionaryEntry': true }, { 'dossierDictionaryEntry': true } ], " +
|
||||
" 'positions': { $in: ?2 } " +
|
||||
"}")
|
||||
List<EntityLogEntryDocument> findDictionaryEntityLogEntriesByValueAndPositions(String entityLogId,
|
||||
String value,
|
||||
List<List<Position>> positionsList);
|
||||
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogDocument;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.exception.DocumentNotFoundException;
|
||||
@ -407,4 +408,17 @@ public class EntityLogMongoService {
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public Set<EntityLogEntry> findDictionaryEntityLogEntriesByValueAndPositions(String dossierId,
|
||||
String fileId,
|
||||
String value,
|
||||
List<List<Position>> positionsList) {
|
||||
|
||||
return new HashSet<>(entityLogEntryDocumentRepository.findDictionaryEntityLogEntriesByValueAndPositions(mapper.getLogId(dossierId, fileId),
|
||||
value,
|
||||
positionsList)
|
||||
.stream()
|
||||
.map(mapper::fromLogEntryDocument)
|
||||
.toList());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user