Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9495 - Remove here for locally resized dictionary entry should remove the entry completly

Browse Source 
- any local resize, legal basis change and recategorization made to a dictionary entry will be unlinked and will create a new id to be used and will introduce 2 manual changes: local add and the manual change.
- unit tests added
This commit is contained in:
corinaolariu 2024-07-10 15:55:57 +03:00
parent 6cdd888967
commit 4fbea2e3e7
15 changed files with 225 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java vendored
View File

@ -71,12 +71,13 @@ public class ManualRedactionController implements ManualRedactionResource {
public void undo(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<String> annotationIds,
@RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
@RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed,
@RequestParam(value = "localOnly", required = false, defaultValue = FALSE) boolean localOnly) {
accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsApprover(dossierId);
manualRedactionUndoService.undo(dossierId, fileId, annotationIds, includeUnprocessed);
manualRedactionUndoService.undo(dossierId, fileId, annotationIds, includeUnprocessed, localOnly);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java
View File

@ -41,6 +41,8 @@ public class ManualLegalBasisChangeEntity implements IBaseAnnotation {
private OffsetDateTime softDeletedTime;
@Column
private int page;
@Column
private String basedOnDictAnnotationId;
@ManyToOne
private FileEntity fileStatus;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRecategorizationEntity.java
View File

@ -56,6 +56,8 @@ public class ManualRecategorizationEntity implements IBaseAnnotation {
private String section;
@Column
private String value;
@Column
private String basedOnDictAnnotationId;
@ManyToOne
private FileEntity fileStatus;

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java
View File

@ -63,6 +63,9 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation {
@Column
private boolean addToAllDossiers;
@Column
private String basedOnDictAnnotationId;
@ElementCollection
@Fetch(value = FetchMode.SUBSELECT)
private Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();

41
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
View File

@ -7,6 +7,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@ -108,19 +109,36 @@ public class EntityLogMergeService {
List<EntityLogEntry> entityLogEntries,
Map<String, List<BaseAnnotation>> allManualChanges) {
Map<String, String> trackLocalChangesBasedOnDictEntriesMap = new HashMap<>();
Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLog, dossier, analysisNumber)//
.collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
entityLogEntries.addAll(addedLocalManualEntries.values());
buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(entityLogEntries::add);
processEntityLogEntries(dossier, entityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges);
processEntityLogEntries(dossier, entityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges, trackLocalChangesBasedOnDictEntriesMap);
adjustEntityLogEntriesAfterLocalChangesBasedOnDict(entityLogEntries, trackLocalChangesBasedOnDictEntriesMap, analysisNumber);
}
private void adjustEntityLogEntriesAfterLocalChangesBasedOnDict(List<EntityLogEntry> entityLogEntries, Map<String, String> trackLocalChangesBasedOnDictEntriesMap, int analysisNumber) {
var dictEntryIdsToUpdate = trackLocalChangesBasedOnDictEntriesMap.values();
entityLogEntries.stream().filter(entityLogEntry -> dictEntryIdsToUpdate.contains(entityLogEntry.getId()))
.forEach(entityLogEntry -> {
List<Change> changes = new ArrayList<>();
changes.add(ChangeFactory.toChange(ChangeType.REMOVED,
OffsetDateTime.now(),
analysisNumber,
PropertyChange.builder().property("state").oldValue(entityLogEntry.getState().name()).newValue(EntryState.REMOVED.name()).build()));
addChanges(entityLogEntry, changes);
entityLogEntry.setState(EntryState.REMOVED);
});
}
private void processEntityLogEntries(DossierEntity dossier,
List<EntityLogEntry> entityLogEntries,
Map<String, EntityLogEntry> addedLocalManualEntries,
int analysisNumber,
Map<String, List<BaseAnnotation>> allManualChanges) {
Map<String, List<BaseAnnotation>> allManualChanges,
Map<String, String> trackLocalChangesBasedOnDictEntriesMap) {
int numberOfAddedPendingEntries = 0; // since list is dynamically growing we need to keep track of the number of added pending entries to ignore them in the loop
for (int i = 0; i + numberOfAddedPendingEntries < entityLogEntries.size(); i++) {
@ -135,7 +153,8 @@ public class EntityLogMergeService {
List<EntityLogEntry> pendingImageRecategorizations = mergeLocalManualChangesAndReturnNonMergeableAsPending(dossier,
allManualChanges,
entityLogEntry,
analysisNumber);
analysisNumber,
trackLocalChangesBasedOnDictEntriesMap);
List<EntityLogEntry> pendingDictionaryEntries = buildPendingDictionaryEntries(allManualChanges, entityLogEntry);
// insert pending entries directly after the associated entry to enable performant linking in UI
@ -226,7 +245,8 @@ public class EntityLogMergeService {
private List<EntityLogEntry> mergeLocalManualChangesAndReturnNonMergeableAsPending(DossierEntity dossier,
Map<String, List<BaseAnnotation>> allManualChanges,
EntityLogEntry entityLogEntry,
int analysisNumber) {
int analysisNumber,
Map<String, String> trackLocalChangesBasedOnDictEntriesMap) {
return allManualChanges.getOrDefault(entityLogEntry.getId(), Collections.emptyList())
.stream()
@ -235,14 +255,27 @@ public class EntityLogMergeService {
.map(localChange -> {
if (localChange instanceof IdRemoval idRemoval) {
mergeIdToRemove(idRemoval, entityLogEntry, analysisNumber);
if (trackLocalChangesBasedOnDictEntriesMap.containsKey(idRemoval.getAnnotationId())) {
trackLocalChangesBasedOnDictEntriesMap.remove(idRemoval.getAnnotationId());
}
return null;
} else if (localChange instanceof ManualResizeRedaction manualResizeRedaction) {
mergeResizeRedaction(manualResizeRedaction, entityLogEntry, analysisNumber);
if (manualResizeRedaction.getBasedOnDictAnnotationId() != null) {
trackLocalChangesBasedOnDictEntriesMap.put(manualResizeRedaction.getAnnotationId(), manualResizeRedaction.getBasedOnDictAnnotationId());
}
return null;
} else if (localChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
mergeLegalBasisChange(manualLegalBasisChange, entityLogEntry, analysisNumber);
if (manualLegalBasisChange.getBasedOnDictAnnotationId() != null) {
trackLocalChangesBasedOnDictEntriesMap.put(manualLegalBasisChange.getAnnotationId(), manualLegalBasisChange.getBasedOnDictAnnotationId());
}
return null;
} else if (localChange instanceof ManualRecategorization manualRecategorization) {
if (manualRecategorization.getBasedOnDictAnnotationId() != null) {
trackLocalChangesBasedOnDictEntriesMap.put(manualRecategorization.getAnnotationId(), manualRecategorization.getBasedOnDictAnnotationId());
}
return mergeRecategorization(manualRecategorization, entityLogEntry, dossier, analysisNumber);
} else if (localChange instanceof ManualForceRedaction manualForceRedaction) {
mergeForceRedaction(manualForceRedaction, entityLogEntry, analysisNumber);

15
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java
View File

@ -6,6 +6,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
import org.springframework.stereotype.Service;
@ -155,6 +156,7 @@ public class ManualRedactionMapper {
for (LegalBasisChangeRequestModel legalBasisChangeRequest : legalBasisChangeRequests) {
String uuid = UUID.randomUUID().toString();
EntityLogEntry entityLogEntry = entityLogMongoWrapperService.getEntityLogEntryById(dossierId, fileId, legalBasisChangeRequest.getAnnotationId());
LegalBasisChangeRequest request = LegalBasisChangeRequest.builder()
.annotationId(legalBasisChangeRequest.getAnnotationId())
@ -166,7 +168,10 @@ public class ManualRedactionMapper {
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
request.setBasedOnDictAnnotationId(legalBasisChangeRequest.getAnnotationId());
entityLogEntry.setId(uuid);
manualRedactionEntryConsumer.accept(entityLogEntry);
request.setAnnotationId(uuid);
}
requests.add(RequestEntryPair.<LegalBasisChangeRequest>builder().request(request).entityLogEntry(entityLogEntry).build());
@ -197,6 +202,7 @@ public class ManualRedactionMapper {
entityLogEntries.forEach(entityLogEntry -> {
String changedValue;
String changedTypeId;
String uuid = UUID.randomUUID().toString();
if (recategorizationRequest.isAddToDictionary() || recategorizationRequest.isAddToAllDossiers()) {
changedValue = recategorizationRequest.getValue();
@ -233,7 +239,10 @@ public class ManualRedactionMapper {
&& !recategorizationRequest.isAddToAllDossiers()
&& !recategorizationRequest.isAddToDictionary()
&& entryIsEntityType(entityLogEntry)) {
request.setBasedOnDictAnnotationId(recategorizationRequest.getAnnotationId());
entityLogEntry.setId(uuid);
manualRedactionEntryConsumer.accept(entityLogEntry);
request.setAnnotationId(uuid);
}
requests.add(RequestEntryPair.<RecategorizationRequest>builder().request(request).entityLogEntry(entityLogEntry).build());
@ -289,6 +298,8 @@ public class ManualRedactionMapper {
for (ResizeRedactionRequestModel resizeRedactionRequest : resizeRedactionRequests) {
entityLogEntries.forEach(entityLogEntry -> {
String uuid = UUID.randomUUID().toString();
ResizeRedactionRequest request = ResizeRedactionRequest.builder()
.annotationId(resizeRedactionRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
@ -300,11 +311,13 @@ public class ManualRedactionMapper {
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL)
&& entryIsEntityType(entityLogEntry)
&& !request.isAddToAllDossiers()
&& !request.getUpdateDictionary()
&& entryIsEntityType(entityLogEntry)) {
request.setBasedOnDictAnnotationId(resizeRedactionRequest.getAnnotationId());
entityLogEntry.setId(uuid);
manualRedactionEntryConsumer.accept(entityLogEntry);
request.setAnnotationId(uuid);
}
requests.add(RequestEntryPair.<ResizeRedactionRequest>builder().request(request).entityLogEntry(entityLogEntry).build());

4
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -214,4 +214,6 @@ databaseChangeLog:
- include:
file: db/changelog/tenant/130-add-primary-key-constraint-download-status-reports.yaml
- include:
file: db/changelog/tenant/131-changed-max-size-for-id-in-entity.yaml
file: db/changelog/tenant/131-changed-max-size-for-id-in-entity.yaml
- include:
file: db/changelog/tenant/132-add-based-on-dict-annotation-id-to-manual_changes.yaml

45
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/132-add-based-on-dict-annotation-id-to-manual_changes.yaml Normal file
View File

@ -0,0 +1,45 @@
databaseChangeLog:
- changeSet:
id: add-annotation-info-to-manual-resize
author: colariu
changes:
- addColumn:
columns:
- column:
name: based_on_dict_annotation_id
type: VARCHAR(255)
tableName: manual_resize_redaction
- changeSet:
id: add-annotation-info-to-manual-legal-basis-change
author: colariu
changes:
- addColumn:
columns:
- column:
name: based_on_dict_annotation_id
type: VARCHAR(255)
tableName: manual_legal_basis_change
- changeSet:
id: add-annotation-info-to-manual-recategorization
author: colariu
changes:
- addColumn:
columns:
- column:
name: based_on_dict_annotation_id
type: VARCHAR(255)
tableName: manual_recategorization
# - changeSet:
# id: add-annotation-info-to-manual-force-redaction
# author: colariu
# changes:
# - addColumn:
# columns:
# - column:
# name: based_on_dict_annotation_id
# type: VARCHAR(255)
# - column:
# name: based_on_dict_entry
# type: BOOLEAN
# defaultValue: false
# tableName: manual_force_redaction

110
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java
View File

@ -378,6 +378,115 @@ public class EntityLogMergeTest {
.get(index + 1).getState(), EntryState.PENDING);
}
//dict entry, resize local
@Test
public void testMergeEntityLogWithManualResizeBasedOnDictRedaction() {
String dossierId = "dossierId";
String dossierTemplateId = "dossierTemplateId";
String entryToRemoveId = UUID.randomUUID().toString();
String dictEntryToResizeId = UUID.randomUUID().toString();
String entryLegalBasisId = UUID.randomUUID().toString();
String forceRedactionId = UUID.randomUUID().toString();
String entryToRecategorizeId = UUID.randomUUID().toString();
String localId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = ManualRedactions.builder()
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
.resizeRedactions(Set.of(provideManualResize(localId, "Darth", dictEntryToResizeId)))
.build();
var entityLog = provideEntityLog(entryToRemoveId, dictEntryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, true);
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
when(fileStatusService.getStatus(FILE_ID)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(FILE_ID).build());
when(fileManagementStorageService.getEntityLog(dossierId, FILE_ID)).thenReturn(entityLog);
when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(FileEntity.builder().id(FILE_ID).fileAttributes(Collections.emptyList()).build());
when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
assertNotNull(response);
assertFalse(response.getEntityLogEntry().isEmpty());
var optionalResizeEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(dictEntryToResizeId))
.findFirst();
assertTrue(optionalResizeEntryLogEntry.isPresent());
assertEquals(EntryType.ENTITY, optionalResizeEntryLogEntry.get().getEntryType());
assertEquals(EntryState.REMOVED, optionalResizeEntryLogEntry.get().getState());
var optionalResizeEntryLogEntry2 = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(localId))
.findFirst();
assertTrue(optionalResizeEntryLogEntry2.isPresent());
assertEquals(EntryType.ENTITY, optionalResizeEntryLogEntry2.get().getEntryType());
assertEquals(EntryState.APPLIED, optionalResizeEntryLogEntry2.get().getState());
assertFalse(optionalResizeEntryLogEntry.get().getId().equals(optionalResizeEntryLogEntry2.get().getId()));
}
//dict entry, resize local, remove local
@Test
public void testMergeEntityLogWithManualResizeAndRemoveChangesOnDictRedaction() {
String dossierId = "dossierId";
String dossierTemplateId = "dossierTemplateId";
String entryToRemoveId = UUID.randomUUID().toString();
String dictEntryToResizeId = UUID.randomUUID().toString();
String entryLegalBasisId = UUID.randomUUID().toString();
String forceRedactionId = UUID.randomUUID().toString();
String entryToRecategorizeId = UUID.randomUUID().toString();
String localId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = ManualRedactions.builder()
.entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
.resizeRedactions(Set.of(provideManualResize(localId, "Darth", dictEntryToResizeId)))
.idsToRemove(Set.of(IdRemoval.builder().annotationId(localId).requestDate(OffsetDateTime.now()).user("user").fileId(FILE_ID).build()))
.build();
var entityLog = provideEntityLog(entryToRemoveId, dictEntryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, true);
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
when(fileStatusService.getStatus(FILE_ID)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(FILE_ID).build());
when(fileManagementStorageService.getEntityLog(dossierId, FILE_ID)).thenReturn(entityLog);
when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(FileEntity.builder().id(FILE_ID).fileAttributes(Collections.emptyList()).build());
when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
assertNotNull(response);
assertFalse(response.getEntityLogEntry().isEmpty());
var optionalResizeEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(dictEntryToResizeId))
.findFirst();
assertTrue(optionalResizeEntryLogEntry.isPresent());
assertEquals(EntryType.ENTITY, optionalResizeEntryLogEntry.get().getEntryType());
assertEquals(EntryState.APPLIED, optionalResizeEntryLogEntry.get().getState());
var optionalResizeEntryLogEntry2 = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(localId))
.findFirst();
assertTrue(optionalResizeEntryLogEntry2.isPresent());
assertEquals(EntryType.ENTITY, optionalResizeEntryLogEntry2.get().getEntryType());
assertEquals(EntryState.REMOVED, optionalResizeEntryLogEntry2.get().getState());
assertFalse(optionalResizeEntryLogEntry.get().getId().equals(optionalResizeEntryLogEntry2.get().getId()));
}
@Test
void testMergeEntityLogWithManualAddAndRemoveChanges() {
@ -638,6 +747,7 @@ public class EntityLogMergeTest {
.addToAllDossiers(false)
.user("User")
.fileId("file")
.basedOnDictAnnotationId(dictId)
.build();
}

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/LegalBasisChangeRequest.java
View File

@ -18,5 +18,6 @@ public class LegalBasisChangeRequest {
private int page;
private String section;
private String value;
private String basedOnDictAnnotationId;
}

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RecategorizationRequest.java
View File

@ -30,6 +30,7 @@ public class RecategorizationRequest implements ManualRequestWithAddToDictionary
private DictionaryEntryType dictionaryEntryType;
String legalBasis;
String section;
private String basedOnDictAnnotationId;
@Override

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ResizeRedactionRequest.java
View File

@ -28,5 +28,6 @@ public class ResizeRedactionRequest {
private String textAfter;
private boolean addToAllDossiers;
private String basedOnDictAnnotationId;
}

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualLegalBasisChange.java
View File

@ -16,6 +16,7 @@ public class ManualLegalBasisChange extends BaseAnnotation {
private String section;
private String value;
private String legalBasis;
private String basedOnDictAnnotationId;
@Override

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualRecategorization.java
View File

@ -19,6 +19,7 @@ public class ManualRecategorization extends BaseAnnotation {
private boolean addToAllDossiers;
private String section;
private String value;
private String basedOnDictAnnotationId;
@Override

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualResizeRedaction.java
View File

@ -24,6 +24,7 @@ public class ManualResizeRedaction extends BaseAnnotation {
private String textAfter;
private Boolean updateDictionary;
private boolean addToAllDossiers;
private String basedOnDictAnnotationId;
@Override