RED-10186 - Unlinked annotation with manual changes still linked and removed in specific corner case
- when a local removal is done the MANUAL engine is not added anymore. - removed basedOnDictAnnotationId from manual changes. The add which is always created at a local change with unlink with save the dictionary annotation id in the source id - unit tests updated
This commit is contained in:
corinaolariu
parent
48a0637fba
commit
f2df7fe783
@ -36,8 +36,6 @@ public class ManualForceRedactionEntity implements IBaseAnnotation {
|
||||
private OffsetDateTime softDeletedTime;
|
||||
@Column
|
||||
private int page;
|
||||
@Column
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
@ManyToOne
|
||||
private FileEntity fileStatus;
|
||||
|
||||
@ -39,8 +39,6 @@ public class ManualLegalBasisChangeEntity implements IBaseAnnotation {
|
||||
private OffsetDateTime softDeletedTime;
|
||||
@Column
|
||||
private int page;
|
||||
@Column
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
@ManyToOne
|
||||
private FileEntity fileStatus;
|
||||
|
||||
@ -54,8 +54,6 @@ public class ManualRecategorizationEntity implements IBaseAnnotation {
|
||||
private String section;
|
||||
@Column
|
||||
private String value;
|
||||
@Column
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
@ManyToOne
|
||||
private FileEntity fileStatus;
|
||||
|
||||
@ -61,9 +61,6 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation {
|
||||
@Column
|
||||
private boolean addToAllDossiers;
|
||||
|
||||
@Column
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
@ElementCollection
|
||||
@Fetch(value = FetchMode.SUBSELECT)
|
||||
private Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
|
||||
|
||||
@ -103,13 +103,17 @@ public class EntityLogMergeService {
|
||||
int analysisNumber,
|
||||
Map<String, List<BaseAnnotation>> allManualChanges) {
|
||||
|
||||
Map<String, String> trackLocalChangesBasedOnDictEntriesMap = new HashMap<>();
|
||||
List<EntityLogEntry> mergedEntityLogEntries = new LinkedList<>(entityLogEntries);
|
||||
Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber)//
|
||||
.collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
|
||||
mergedEntityLogEntries.addAll(addedLocalManualEntries.values());
|
||||
buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(mergedEntityLogEntries::add);
|
||||
|
||||
Map<String, String> trackLocalChangesBasedOnDictEntriesMap = unprocessedManualRedactions.getEntriesToAdd()
|
||||
.stream()
|
||||
.filter(ManualRedactionEntry::isLocal)
|
||||
.filter(entry -> entry.getSourceId() != null && !entry.getSourceId().isEmpty())
|
||||
.collect(Collectors.toMap(ManualRedactionEntry::getAnnotationId, ManualRedactionEntry::getSourceId));
|
||||
processEntityLogEntries(dossier, mergedEntityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges, trackLocalChangesBasedOnDictEntriesMap);
|
||||
|
||||
adjustEntityLogEntriesAfterLocalChangesBasedOnDict(entityLogEntries, trackLocalChangesBasedOnDictEntriesMap, analysisNumber);
|
||||
@ -268,26 +272,14 @@ public class EntityLogMergeService {
|
||||
return null;
|
||||
} else if (localChange instanceof ManualResizeRedaction manualResizeRedaction) {
|
||||
mergeResizeRedaction(manualResizeRedaction, entityLogEntry, analysisNumber);
|
||||
if (manualResizeRedaction.getBasedOnDictAnnotationId() != null) {
|
||||
trackLocalChangesBasedOnDictEntriesMap.put(manualResizeRedaction.getAnnotationId(), manualResizeRedaction.getBasedOnDictAnnotationId());
|
||||
}
|
||||
|
||||
return null;
|
||||
} else if (localChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
|
||||
mergeLegalBasisChange(manualLegalBasisChange, entityLogEntry, analysisNumber);
|
||||
if (manualLegalBasisChange.getBasedOnDictAnnotationId() != null) {
|
||||
trackLocalChangesBasedOnDictEntriesMap.put(manualLegalBasisChange.getAnnotationId(), manualLegalBasisChange.getBasedOnDictAnnotationId());
|
||||
}
|
||||
return null;
|
||||
} else if (localChange instanceof ManualRecategorization manualRecategorization) {
|
||||
if (manualRecategorization.getBasedOnDictAnnotationId() != null) {
|
||||
trackLocalChangesBasedOnDictEntriesMap.put(manualRecategorization.getAnnotationId(), manualRecategorization.getBasedOnDictAnnotationId());
|
||||
}
|
||||
return mergeRecategorization(manualRecategorization, entityLogEntry, dossier, analysisNumber);
|
||||
} else if (localChange instanceof ManualForceRedaction manualForceRedaction) {
|
||||
if (manualForceRedaction.getBasedOnDictAnnotationId() != null) {
|
||||
trackLocalChangesBasedOnDictEntriesMap.put(manualForceRedaction.getAnnotationId(), manualForceRedaction.getBasedOnDictAnnotationId());
|
||||
}
|
||||
mergeForceRedaction(manualForceRedaction, entityLogEntry, analysisNumber);
|
||||
return null;
|
||||
} else {
|
||||
@ -393,11 +385,10 @@ public class EntityLogMergeService {
|
||||
|
||||
entityLogEntry.setState(EntryState.IGNORED);
|
||||
//special case, only for add local and remove only
|
||||
if (entityLogEntry.getEngines().equals(Set.of(Engine.MANUAL))) {
|
||||
if (entityLogEntry.getEngines().contains(Engine.MANUAL)) {
|
||||
entityLogEntry.setState(EntryState.REMOVED);
|
||||
change.setType(ChangeType.REMOVED);
|
||||
}
|
||||
entityLogEntry.getEngines().add(Engine.MANUAL);
|
||||
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(idRemoval, 0));
|
||||
|
||||
changes.add(change);
|
||||
@ -435,7 +426,7 @@ public class EntityLogMergeService {
|
||||
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
|
||||
entityLogEntry.setPositions(newPositions);
|
||||
entityLogEntry.setValue(manualResizeRedaction.getValue());
|
||||
entityLogEntry.getEngines().add(Engine.MANUAL);
|
||||
// entityLogEntry.getEngines().add(Engine.MANUAL); //TODO: is this needed?
|
||||
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualResizeRedaction, 0));
|
||||
}
|
||||
|
||||
@ -464,7 +455,7 @@ public class EntityLogMergeService {
|
||||
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
|
||||
entityLogEntry.setSection(manualLegalBasisChange.getSection());
|
||||
entityLogEntry.setValue(manualLegalBasisChange.getValue());
|
||||
entityLogEntry.getEngines().add(Engine.MANUAL);
|
||||
// entityLogEntry.getEngines().add(Engine.MANUAL); //TODO: is this needed?
|
||||
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualLegalBasisChange, 0));
|
||||
}
|
||||
|
||||
@ -486,7 +477,7 @@ public class EntityLogMergeService {
|
||||
return pendingEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
|
||||
}
|
||||
|
||||
entityLogEntry.getEngines().add(Engine.MANUAL);
|
||||
// entityLogEntry.getEngines().add(Engine.MANUAL); //TODO: is this needed?
|
||||
|
||||
if (recategorization.getType() != null && !recategorization.getType().equals(entityLogEntry.getType())) {
|
||||
boolean isHint = isHint(recategorization.getType(), dossier);
|
||||
@ -549,10 +540,7 @@ public class EntityLogMergeService {
|
||||
PropertyChange.builder().property("state").oldValue(oldState.name()).newValue(newState.name()).build()));
|
||||
entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
|
||||
entityLogEntry.setState(newState);
|
||||
entityLogEntry.getEngines().add(Engine.MANUAL);
|
||||
if (forceRedaction.getBasedOnDictAnnotationId() != null) {
|
||||
entityLogEntry.getEngines().add(Engine.DICTIONARY);
|
||||
}
|
||||
// entityLogEntry.getEngines().add(Engine.MANUAL); //TODO: is this needed?
|
||||
addChanges(entityLogEntry, changes);
|
||||
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(forceRedaction, 0));
|
||||
}
|
||||
|
||||
@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionEntrySource;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
|
||||
@ -138,7 +139,7 @@ public class ManualRedactionMapper {
|
||||
public List<RequestEntryPair<ForceRedactionRequest>> toForceRedactionRequestList(String dossierId,
|
||||
String fileId,
|
||||
Set<ForceRedactionRequestModel> forceRedactionRequests,
|
||||
Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
|
||||
Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
|
||||
|
||||
List<RequestEntryPair<ForceRedactionRequest>> requests = new ArrayList<>();
|
||||
|
||||
@ -154,9 +155,8 @@ public class ManualRedactionMapper {
|
||||
.build();
|
||||
|
||||
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !entityLogEntry.getEngines().contains(Engine.IMPORTED) && entryIsEntityType(entityLogEntry)) {
|
||||
request.setBasedOnDictAnnotationId(forceRedactionRequestModel.getAnnotationId());
|
||||
entityLogEntry.setId(uuid);
|
||||
manualRedactionEntryConsumer.accept(entityLogEntry);
|
||||
manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, forceRedactionRequestModel.getAnnotationId()));
|
||||
request.setAnnotationId(uuid);
|
||||
}
|
||||
|
||||
@ -170,7 +170,7 @@ public class ManualRedactionMapper {
|
||||
public List<RequestEntryPair<LegalBasisChangeRequest>> toLegalBasisChangeRequestList(String dossierId,
|
||||
String fileId,
|
||||
Set<LegalBasisChangeRequestModel> legalBasisChangeRequests,
|
||||
Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
|
||||
Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
|
||||
|
||||
List<RequestEntryPair<LegalBasisChangeRequest>> requests = new ArrayList<>();
|
||||
|
||||
@ -188,9 +188,8 @@ public class ManualRedactionMapper {
|
||||
.build();
|
||||
|
||||
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !entityLogEntry.getEngines().contains(Engine.IMPORTED) && entryIsEntityType(entityLogEntry)) {
|
||||
request.setBasedOnDictAnnotationId(legalBasisChangeRequest.getAnnotationId());
|
||||
entityLogEntry.setId(uuid);
|
||||
manualRedactionEntryConsumer.accept(entityLogEntry);
|
||||
manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, legalBasisChangeRequest.getAnnotationId()));
|
||||
request.setAnnotationId(uuid);
|
||||
}
|
||||
|
||||
@ -206,7 +205,7 @@ public class ManualRedactionMapper {
|
||||
String dossierTemplateId,
|
||||
Set<RecategorizationRequestModel> recategorizationRequests,
|
||||
boolean includeUnprocessed,
|
||||
Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
|
||||
Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
|
||||
|
||||
List<EntityLogEntry> entityLogEntries = entityLogMongoWrapperService.getEntityLogEntriesByIds(dossierId,
|
||||
fileId,
|
||||
@ -273,9 +272,8 @@ public class ManualRedactionMapper {
|
||||
&& !recategorizationRequest.isAddToAllDossiers()
|
||||
&& !recategorizationRequest.isAddToDictionary()
|
||||
&& entryIsEntityType(entityLogEntry)) {
|
||||
request.setBasedOnDictAnnotationId(recategorizationRequest.getAnnotationId());
|
||||
entityLogEntry.setId(uuid);
|
||||
manualRedactionEntryConsumer.accept(entityLogEntry);
|
||||
manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, recategorizationRequest.getAnnotationId()));
|
||||
request.setAnnotationId(uuid);
|
||||
}
|
||||
|
||||
@ -317,7 +315,7 @@ public class ManualRedactionMapper {
|
||||
public List<RequestEntryPair<ResizeRedactionRequest>> toResizeRedactionRequestList(String dossierId,
|
||||
String fileId,
|
||||
Set<ResizeRedactionRequestModel> resizeRedactionRequests,
|
||||
Consumer<EntityLogEntry> manualRedactionEntryConsumer,
|
||||
Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer,
|
||||
boolean includeUnprocessed) {
|
||||
|
||||
List<EntityLogEntry> entityLogEntries = entityLogMongoWrapperService.getEntityLogEntriesByIds(dossierId,
|
||||
@ -352,9 +350,8 @@ public class ManualRedactionMapper {
|
||||
&& !request.isAddToAllDossiers()
|
||||
&& !request.getUpdateDictionary()
|
||||
&& entryIsEntityType(entityLogEntry)) {
|
||||
request.setBasedOnDictAnnotationId(resizeRedactionRequest.getAnnotationId());
|
||||
entityLogEntry.setId(uuid);
|
||||
manualRedactionEntryConsumer.accept(entityLogEntry);
|
||||
manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, resizeRedactionRequest.getAnnotationId()));
|
||||
request.setAnnotationId(uuid);
|
||||
}
|
||||
|
||||
|
||||
@ -58,6 +58,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionEntrySource;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
@ -637,8 +638,9 @@ public class ManualRedactionService {
|
||||
}
|
||||
|
||||
|
||||
private void addManualRedactionEntry(String fileId, EntityLogEntry entityLogEntry) {
|
||||
private void addManualRedactionEntry(String fileId, ManualRedactionEntrySource manualRedactionEntrySource) {
|
||||
|
||||
EntityLogEntry entityLogEntry = manualRedactionEntrySource.getEntityLogEntry();
|
||||
ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
|
||||
.value(entityLogEntry.getValue())
|
||||
.reason(entityLogEntry.getReason())
|
||||
@ -656,6 +658,7 @@ public class ManualRedactionService {
|
||||
.dictionaryEntryType(getDictionaryEntryType(entityLogEntry))
|
||||
.fileId(fileId)
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.sourceId(manualRedactionEntrySource.getSourceId())
|
||||
.build();
|
||||
|
||||
addManualRedactionEntries(List.of(manualRedactionEntry), false);
|
||||
@ -670,7 +673,7 @@ public class ManualRedactionService {
|
||||
}
|
||||
|
||||
|
||||
private Consumer<EntityLogEntry> getEntityLogEntryConsumer(String fileId) {
|
||||
private Consumer<ManualRedactionEntrySource> getEntityLogEntryConsumer(String fileId) {
|
||||
|
||||
return entry -> addManualRedactionEntry(fileId, entry);
|
||||
}
|
||||
|
||||
@ -237,3 +237,5 @@ databaseChangeLog:
|
||||
file: db/changelog/tenant/146-add-layout-parsing-type-to-dossier-template.yaml
|
||||
- include:
|
||||
file: db/changelog/tenant/147-add-quotechar-to-component-mapping.yaml
|
||||
- include:
|
||||
file: db/changelog/tenant/148-remove-based-on-dict-annotation-id-columns.yaml
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
databaseChangeLog:
|
||||
- changeSet:
|
||||
id: remove-based-on-dict-annotation-id-column
|
||||
author: corina
|
||||
changes:
|
||||
- dropColumn:
|
||||
columns:
|
||||
- name: based_on_dict_annotation_id
|
||||
tableName: manual_force_redaction
|
||||
- dropColumn:
|
||||
columns:
|
||||
- name: based_on_dict_annotation_id
|
||||
tableName: manual_resize_redaction
|
||||
- dropColumn:
|
||||
columns:
|
||||
- name: based_on_dict_annotation_id
|
||||
tableName: manual_legal_basis_change
|
||||
- dropColumn:
|
||||
columns:
|
||||
- name: based_on_dict_annotation_id
|
||||
tableName: manual_recategorization
|
||||
@ -173,7 +173,7 @@ public class EntityLogMergeTest {
|
||||
.get(0).getManualRedactionType(), ManualRedactionType.REMOVE);
|
||||
assertEquals(removeEntryLogEntry.getChanges()
|
||||
.get(0).getType(), ChangeType.IGNORED);
|
||||
assertTrue(removeEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
assertFalse(removeEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
|
||||
var optionalResizeEntryLogEntry = response.getEntityLogEntry()
|
||||
.stream()
|
||||
@ -203,7 +203,7 @@ public class EntityLogMergeTest {
|
||||
assertEquals(resizeEntryLogEntry.getChanges()
|
||||
.get(0).getPropertyChanges()
|
||||
.get("positions"), "[1.0, 1.0, 1.0, 1.0, 1] -> [2.0, 2.0, 2.0, 2.0, 1]");
|
||||
assertTrue(resizeEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
// assertTrue(resizeEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
|
||||
var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry()
|
||||
.stream()
|
||||
@ -221,7 +221,7 @@ public class EntityLogMergeTest {
|
||||
assertEquals(legalBasisEntryLogEntry.getChanges()
|
||||
.get(0).getPropertyChanges()
|
||||
.get("legalBasis"), "legalBasis -> New legal basis");
|
||||
assertTrue(legalBasisEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
// assertTrue(legalBasisEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
|
||||
var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry()
|
||||
.stream()
|
||||
@ -242,7 +242,7 @@ public class EntityLogMergeTest {
|
||||
assertEquals(forceRedactionEntryLogEntry.getChanges()
|
||||
.get(0).getPropertyChanges()
|
||||
.get("state"), "SKIPPED -> APPLIED");
|
||||
assertTrue(forceRedactionEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
// assertTrue(forceRedactionEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
|
||||
var optionalRectangleEntryLogEntry = response.getEntityLogEntry()
|
||||
.stream()
|
||||
@ -278,7 +278,7 @@ public class EntityLogMergeTest {
|
||||
assertEquals(recategorizationEntryLogEntry.getChanges()
|
||||
.get(0).getPropertyChanges()
|
||||
.get("type"), "CBI_author -> PII");
|
||||
assertTrue(recategorizationEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
// assertTrue(recategorizationEntryLogEntry.getEngines().contains(Engine.MANUAL));
|
||||
}
|
||||
|
||||
|
||||
@ -419,8 +419,8 @@ public class EntityLogMergeTest {
|
||||
String localId = UUID.randomUUID().toString();
|
||||
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder()
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
|
||||
.resizeRedactions(Set.of(provideManualResize(localId, "Darth", dictEntryToResizeId)))
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", dictEntryToResizeId)))
|
||||
.resizeRedactions(Set.of(provideManualResize(localId, "Darth")))
|
||||
.build();
|
||||
|
||||
var entityLog = provideEntityLog(entryToRemoveId, dictEntryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, true);
|
||||
@ -457,6 +457,59 @@ public class EntityLogMergeTest {
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMergeEntityLogWithManualRecatBasedOnDictRedaction() {
|
||||
|
||||
String dossierId = "dossierId";
|
||||
String dossierTemplateId = "dossierTemplateId";
|
||||
|
||||
String entryToRemoveId = UUID.randomUUID().toString();
|
||||
String dictEntryToResizeId = UUID.randomUUID().toString();
|
||||
String entryLegalBasisId = UUID.randomUUID().toString();
|
||||
String forceRedactionId = UUID.randomUUID().toString();
|
||||
String entryToRecategorizeId = UUID.randomUUID().toString();
|
||||
|
||||
String localId = UUID.randomUUID().toString();
|
||||
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder()
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", entryToRecategorizeId)))
|
||||
.recategorizations(Set.of(provideManualRecat(localId, "Darth Vader")))
|
||||
.build();
|
||||
|
||||
var entityLog = provideEntityLog(entryToRemoveId, dictEntryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, true);
|
||||
|
||||
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
|
||||
when(fileStatusService.getStatus(FILE_ID)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(FILE_ID).build());
|
||||
when(fileManagementStorageService.getEntityLog(dossierId, FILE_ID)).thenReturn(entityLog);
|
||||
when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
|
||||
when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
|
||||
when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(FileEntity.builder().id(FILE_ID).fileAttributes(Collections.emptyList()).build());
|
||||
when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
|
||||
|
||||
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
|
||||
|
||||
assertNotNull(response);
|
||||
assertFalse(response.getEntityLogEntry().isEmpty());
|
||||
|
||||
var optionalRecatEntryLogEntry = response.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRecategorizeId))
|
||||
.findFirst();
|
||||
assertTrue(optionalRecatEntryLogEntry.isPresent());
|
||||
assertEquals(EntryType.ENTITY, optionalRecatEntryLogEntry.get().getEntryType());
|
||||
assertEquals(EntryState.REMOVED, optionalRecatEntryLogEntry.get().getState());
|
||||
|
||||
var optionalRecatEntryLogEntry2 = response.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(entityLogEntry -> entityLogEntry.getId().equals(localId))
|
||||
.findFirst();
|
||||
assertTrue(optionalRecatEntryLogEntry2.isPresent());
|
||||
assertEquals(EntryType.ENTITY, optionalRecatEntryLogEntry2.get().getEntryType());
|
||||
assertEquals(EntryState.APPLIED, optionalRecatEntryLogEntry2.get().getState());
|
||||
assertFalse(optionalRecatEntryLogEntry.get().getId().equals(optionalRecatEntryLogEntry2.get().getId()));
|
||||
}
|
||||
|
||||
|
||||
//dict entry, resize local, remove local
|
||||
@Test
|
||||
public void testMergeEntityLogWithManualResizeAndRemoveChangesOnDictRedaction() {
|
||||
@ -473,8 +526,8 @@ public class EntityLogMergeTest {
|
||||
String localId = UUID.randomUUID().toString();
|
||||
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder()
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
|
||||
.resizeRedactions(Set.of(provideManualResize(localId, "Darth", dictEntryToResizeId)))
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", dictEntryToResizeId)))
|
||||
.resizeRedactions(Set.of(provideManualResize(localId, "Darth")))
|
||||
.idsToRemove(Set.of(IdRemoval.builder().annotationId(localId).requestDate(OffsetDateTime.now()).user("user").fileId(FILE_ID).build()))
|
||||
.build();
|
||||
|
||||
@ -528,7 +581,7 @@ public class EntityLogMergeTest {
|
||||
String localId = UUID.randomUUID().toString();
|
||||
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder()
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", null)))
|
||||
.idsToRemove(Set.of(IdRemoval.builder().annotationId(localId).requestDate(OffsetDateTime.now()).user("user").fileId(FILE_ID).build()))
|
||||
.build();
|
||||
|
||||
@ -573,8 +626,8 @@ public class EntityLogMergeTest {
|
||||
String localId = UUID.randomUUID().toString();
|
||||
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder()
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
|
||||
.resizeRedactions(Set.of(provideManualResize(localId, "Darth", dictEntryToResizeId)))
|
||||
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", dictEntryToResizeId)))
|
||||
.resizeRedactions(Set.of(provideManualResize(localId, "Darth")))
|
||||
.idsToRemove(Set.of(IdRemoval.builder().annotationId(localId).requestDate(OffsetDateTime.now()).user("user").fileId(FILE_ID).build()))
|
||||
.build();
|
||||
|
||||
@ -725,7 +778,8 @@ public class EntityLogMergeTest {
|
||||
.findFirst();
|
||||
assertTrue(optionalEntityLogEntry.isPresent());
|
||||
assertEquals(optionalEntityLogEntry.get().getChanges().size(), 1);
|
||||
assertEquals(optionalEntityLogEntry.get().getChanges().get(0).getType(), ChangeType.IGNORED);
|
||||
assertEquals(optionalEntityLogEntry.get().getChanges()
|
||||
.get(0).getType(), ChangeType.IGNORED);
|
||||
}
|
||||
|
||||
|
||||
@ -877,7 +931,7 @@ public class EntityLogMergeTest {
|
||||
}
|
||||
|
||||
|
||||
private ManualRedactionEntry provideManualAdd(String entryToAddId, String valueToAdd) {
|
||||
private ManualRedactionEntry provideManualAdd(String entryToAddId, String valueToAdd, String sourceId) {
|
||||
|
||||
return ManualRedactionEntry.builder()
|
||||
.positions(List.of(new Rectangle(1f, 2f, 3f, 4f, 1)))
|
||||
@ -892,11 +946,12 @@ public class EntityLogMergeTest {
|
||||
.dictionaryEntryType(DictionaryEntryType.ENTRY)
|
||||
.type("manual")
|
||||
.user("User")
|
||||
.sourceId(sourceId)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private ManualResizeRedaction provideManualResize(String entryToResizeId, String resizedValue, String dictId) {
|
||||
private ManualResizeRedaction provideManualResize(String entryToResizeId, String resizedValue) {
|
||||
|
||||
List<Rectangle> positions = new ArrayList<>();
|
||||
positions.add(new Rectangle(2, 2, 2, 2, 1));
|
||||
@ -910,9 +965,180 @@ public class EntityLogMergeTest {
|
||||
.addToAllDossiers(false)
|
||||
.user("User")
|
||||
.fileId("file")
|
||||
.basedOnDictAnnotationId(dictId)
|
||||
.build();
|
||||
}
|
||||
|
||||
private ManualRecategorization provideManualRecat(String entryToRecatId, String value) {
|
||||
|
||||
return ManualRecategorization.builder()
|
||||
.fileId(FILE_ID)
|
||||
.annotationId(entryToRecatId)
|
||||
.value(value)
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.section("")
|
||||
.addToDictionary(false)
|
||||
.addToAllDossiers(false)
|
||||
.user("User")
|
||||
.fileId("file")
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testRecategorizeWithoutCreatingOverlap() {
|
||||
|
||||
String fileId = "fileId";
|
||||
String dossierId = "dossierId";
|
||||
String dossierTemplateId = "dossierTemplateId";
|
||||
|
||||
String entryId = UUID.randomUUID().toString();
|
||||
String dictId = UUID.randomUUID().toString();
|
||||
|
||||
ManualRedactionEntry add = provideManualAdd(entryId, "Sensitive information", dictId);
|
||||
add.setType("CBI_author");
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder()
|
||||
.entriesToAdd(Set.of(add))
|
||||
.recategorizations(Set.of(ManualRecategorization.builder()
|
||||
.annotationId(entryId)
|
||||
.fileId(fileId)
|
||||
.legalBasis("New legal basis")
|
||||
.type("PII")
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.user("User")
|
||||
.build()))
|
||||
.build();
|
||||
|
||||
List<Position> positions = new ArrayList<>();
|
||||
positions.add(new Position(1, 1, 1, 1, 1));
|
||||
|
||||
EntityLogEntry existingEntry = EntityLogEntry.builder()
|
||||
.id(dictId)
|
||||
.type("CBI_author")
|
||||
.value("Sensitive Information")
|
||||
.entryType(EntryType.ENTITY)
|
||||
.state(EntryState.APPLIED)
|
||||
.dictionaryEntry(true)
|
||||
.positions(positions)
|
||||
.legalBasis("Initial legal basis")
|
||||
.build();
|
||||
|
||||
EntityLog entityLog = new EntityLog(1, 1, Lists.newArrayList(existingEntry), Collections.emptyList(), 0, 0, 0, 0);
|
||||
|
||||
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
|
||||
when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
|
||||
when(fileManagementStorageService.getEntityLog(dossierId, fileId)).thenReturn(entityLog);
|
||||
when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
|
||||
when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
|
||||
when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
|
||||
when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
|
||||
|
||||
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
|
||||
|
||||
assertNotNull(response, "The merged EntityLog should not be null.");
|
||||
assertFalse(response.getEntityLogEntry().isEmpty(), "The merged EntityLog should contain entries.");
|
||||
|
||||
var updatedEntryOpt = response.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getId().equals(entryId))
|
||||
.findFirst();
|
||||
|
||||
assertTrue(updatedEntryOpt.isPresent(), "The recategorized entry should be present in the EntityLog.");
|
||||
|
||||
EntityLogEntry updatedEntry = updatedEntryOpt.get();
|
||||
|
||||
assertEquals("PII", updatedEntry.getType(), "The entry type should be updated to PII.");
|
||||
|
||||
assertEquals("New legal basis", updatedEntry.getLegalBasis(), "The legal basis should be updated.");
|
||||
|
||||
assertEquals(EntryState.APPLIED, updatedEntry.getState(), "The entry state should remain APPLIED.");
|
||||
|
||||
assertEquals(1,
|
||||
response.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(entityLogEntry -> !entityLogEntry.getState().equals(EntryState.REMOVED))
|
||||
.count(),
|
||||
"There should be only one entry in the EntityLog after recategorization.");
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testForceRedactSkippedCBIAddressAndLocalRemoveWithoutCreatingDuplicate() {
|
||||
|
||||
String fileId = "fileId";
|
||||
String dossierId = "dossierId";
|
||||
String dossierTemplateId = "dossierTemplateId";
|
||||
|
||||
String entryId = UUID.randomUUID().toString();
|
||||
String dictId = UUID.randomUUID().toString();
|
||||
|
||||
ManualRedactionEntry add = provideManualAdd(entryId, "123 Main St", dictId);
|
||||
add.setType("CBI_address");
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder()
|
||||
.entriesToAdd(Set.of(add))
|
||||
.forceRedactions(Set.of(ManualForceRedaction.builder()
|
||||
.annotationId(entryId)
|
||||
.fileId(fileId)
|
||||
.legalBasis("Force")
|
||||
.user("User")
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.build()))
|
||||
.idsToRemove(Set.of(IdRemoval.builder().annotationId(entryId).fileId(fileId).user("User").requestDate(OffsetDateTime.now()).build()))
|
||||
.build();
|
||||
|
||||
List<Position> positions = new ArrayList<>();
|
||||
positions.add(new Position(1, 1, 1, 1, 1));
|
||||
|
||||
EntityLogEntry existingEntry = EntityLogEntry.builder()
|
||||
.id(dictId)
|
||||
.type("CBI_address")
|
||||
.value("123 Main St")
|
||||
.entryType(EntryType.ENTITY)
|
||||
.state(EntryState.SKIPPED)
|
||||
.dictionaryEntry(true)
|
||||
.positions(positions)
|
||||
.legalBasis("Initial legal basis")
|
||||
.build();
|
||||
|
||||
EntityLog entityLog = new EntityLog(1, 1, Lists.newArrayList(existingEntry), Collections.emptyList(), 0, 0, 0, 0);
|
||||
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
|
||||
when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
|
||||
when(fileManagementStorageService.getEntityLog(dossierId, fileId)).thenReturn(entityLog);
|
||||
when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
|
||||
when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
|
||||
when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
|
||||
when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
|
||||
|
||||
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
|
||||
|
||||
assertNotNull(response, "The merged EntityLog should not be null.");
|
||||
assertFalse(response.getEntityLogEntry().isEmpty(), "The merged EntityLog should contain entries.");
|
||||
|
||||
List<EntityLogEntry> relatedEntries = response.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getId().equals(entryId))
|
||||
.toList();
|
||||
|
||||
assertEquals(1, relatedEntries.size(), "There should be only one entry for the entryId after force redaction and local removal.");
|
||||
|
||||
EntityLogEntry updatedEntry = relatedEntries.get(0);
|
||||
|
||||
assertEquals("CBI_address", updatedEntry.getType(), "The entry type should remain CBI_address after force redaction and local removal.");
|
||||
|
||||
long skippedAnnotations = response.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getType().equals("manual") && e.getEntryType() == EntryType.AREA && e.getState() == EntryState.SKIPPED)
|
||||
.count();
|
||||
|
||||
assertEquals(0, skippedAnnotations, "No additional skipped annotations (Cross Marks) should be present.");
|
||||
|
||||
var nonRemovedEntries = response.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> !e.getId().equals(entryId) && !e.getState().equals(EntryState.REMOVED))
|
||||
.toList();
|
||||
|
||||
assertEquals(1, nonRemovedEntries.size(), "There should be only one entry in the EntityLog after force and local remove.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -493,7 +493,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
|
||||
AnnotationComments annotationComments = manualRedactionClient.getComments(dossierId, fileId, commentId);
|
||||
assertThat(annotationComments.getComments()).hasSize(1);
|
||||
AnnotationComments annotationCommentsForManualRedactions = manualRedactionClient.getComments(dossierId, fileId, annotationId);
|
||||
assertThat(annotationCommentsForManualRedactions.getComments()).hasSize(2);
|
||||
assertThat(annotationCommentsForManualRedactions.getComments()).hasSize(1);
|
||||
|
||||
fileManagementClient.deleteFile(dossier.getId(), file.getId());
|
||||
var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
|
||||
|
||||
@ -2998,16 +2998,31 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
|
||||
var file = fileTesterAndProvider.testAndProvideFile(dossier);
|
||||
|
||||
var type = typeProvider.testAndProvideType(dossierTemplate);
|
||||
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("Luke Skywalker"), false, dossier.getId(), DictionaryEntryType.ENTRY);
|
||||
String dictValue = "Crandu Seku Laku Meku";
|
||||
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of(dictValue), false, dossier.getId(), DictionaryEntryType.ENTRY);
|
||||
var entityLog = new EntityLog(1,
|
||||
1,
|
||||
List.of(EntityLogEntry.builder()
|
||||
.id("AnnotationId")
|
||||
.type("test")
|
||||
.value("Luke Skywalker")
|
||||
.id("AnnotationId1")
|
||||
.type("CBI_author")
|
||||
.value(dictValue)
|
||||
.entryType(EntryType.ENTITY)
|
||||
.state(EntryState.APPLIED)
|
||||
.dictionaryEntry(true)
|
||||
.dossierDictionaryEntry(true)
|
||||
.engines(Set.of(Engine.DOSSIER_DICTIONARY))
|
||||
.positions(List.of(new Position(new float[]{57f, 446f, 121f, 13f}, 1)))
|
||||
.build(),
|
||||
EntityLogEntry.builder()
|
||||
.id("AnnotationId2")
|
||||
.type("CBI_author")
|
||||
.value(dictValue)
|
||||
.entryType(EntryType.ENTITY)
|
||||
.state(EntryState.APPLIED)
|
||||
.dictionaryEntry(true)
|
||||
.dossierDictionaryEntry(true)
|
||||
.engines(Set.of(Engine.DOSSIER_DICTIONARY))
|
||||
.positions(List.of(new Position(new float[]{57f, 473f, 121f, 13f}, 1)))
|
||||
.build()),
|
||||
null,
|
||||
0,
|
||||
@ -3020,18 +3035,19 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
|
||||
ManualAnnotationResponse response = manualRedactionClient.removeRedactionBulk(dossier.getId(),
|
||||
file.getId(),
|
||||
Set.of(RemoveRedactionRequestModel.builder()
|
||||
.annotationId("AnnotationId")
|
||||
.annotationId("AnnotationId1")
|
||||
.removeFromDictionary(false)
|
||||
.removeFromAllDossiers(false)
|
||||
.build())).getManualAnnotationResponses()
|
||||
.get(0);
|
||||
|
||||
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
|
||||
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId1");
|
||||
assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED);
|
||||
assertEquals(response.getEntityLogEntry().getManualChanges()
|
||||
.get(0).getManualRedactionType(), ManualRedactionType.REMOVE);
|
||||
assertNull(response.getEntityLogEntry().getManualChanges()
|
||||
.get(0).getProcessedDate());
|
||||
assertFalse(response.getEntityLogEntry().getEngines().contains(Engine.MANUAL));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -16,6 +16,5 @@ public class ForceRedactionRequest {
|
||||
private String legalBasis;
|
||||
private String comment;
|
||||
private int page;
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
}
|
||||
|
||||
@ -18,6 +18,5 @@ public class LegalBasisChangeRequest {
|
||||
private int page;
|
||||
private String section;
|
||||
private String value;
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
package com.iqser.red.service.persistence.service.v1.api.shared.model.annotations;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
@Data
|
||||
@SuperBuilder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ManualRedactionEntrySource {
|
||||
|
||||
private EntityLogEntry entityLogEntry;
|
||||
private String sourceId;
|
||||
}
|
||||
|
||||
@ -30,8 +30,6 @@ public class RecategorizationRequest implements ManualRequestWithAddToDictionary
|
||||
private DictionaryEntryType dictionaryEntryType;
|
||||
String legalBasis;
|
||||
String section;
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
|
||||
@Override
|
||||
public boolean isForceAddToDictionary() {
|
||||
|
||||
@ -28,6 +28,5 @@ public class ResizeRedactionRequest {
|
||||
private String textAfter;
|
||||
|
||||
private boolean addToAllDossiers;
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
}
|
||||
|
||||
@ -14,7 +14,6 @@ import lombok.experimental.SuperBuilder;
|
||||
public class ManualForceRedaction extends BaseAnnotation {
|
||||
|
||||
private String legalBasis;
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
|
||||
@Override
|
||||
|
||||
@ -16,7 +16,6 @@ public class ManualLegalBasisChange extends BaseAnnotation {
|
||||
private String section;
|
||||
private String value;
|
||||
private String legalBasis;
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
|
||||
@Override
|
||||
|
||||
@ -19,7 +19,6 @@ public class ManualRecategorization extends BaseAnnotation {
|
||||
private boolean addToAllDossiers;
|
||||
private String section;
|
||||
private String value;
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
|
||||
@Override
|
||||
|
||||
@ -24,7 +24,6 @@ public class ManualResizeRedaction extends BaseAnnotation {
|
||||
private String textAfter;
|
||||
private Boolean updateDictionary;
|
||||
private boolean addToAllDossiers;
|
||||
private String basedOnDictAnnotationId;
|
||||
|
||||
|
||||
@Override
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user