From 2ad5232e4195a5ee6f5028dc8d2d8ab22aa8b95e Mon Sep 17 00:00:00 2001 From: corinaolariu Date: Wed, 28 Aug 2024 11:11:46 +0300 Subject: [PATCH 1/5] RED-9495 - Remove here for locally resized dictionary entry should remove the entry completly - any force made to a dictionary entry will be unlinked and will create a new id to be used and will introduce 2 manual changes: local add and the manual change. - unit tests updated --- .../ManualForceRedactionEntity.java | 2 ++ .../service/EntityLogMergeService.java | 8 +++++++- .../ManualRedactionMapper.java | 4 ++++ .../db/changelog/db.changelog-tenant.yaml | 4 +++- ...annotation-id-to-manual_force_changes.yaml | 11 ++++++++++ .../tests/ManualRedactionTest.java | 20 +++++++++++++++---- .../annotations/ForceRedactionRequest.java | 1 + .../entitymapped/ManualForceRedaction.java | 1 + 8 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java index 8bfa1beaa..71e12b278 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java @@ -36,6 +36,8 @@ public class ManualForceRedactionEntity implements IBaseAnnotation { private OffsetDateTime softDeletedTime; @Column private int page; + @Column + private String basedOnDictAnnotationId; @ManyToOne private FileEntity fileStatus; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java index 8049439db..16216f466 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java @@ -75,7 +75,10 @@ public class EntityLogMergeService { @Observed(name = "EntityLogMergeService", contextualName = "merge-entity-log") - public EntityLog mergeEntityLog(ManualRedactions unprocessedManualRedactions, EntityLog entityLog, DossierEntity dossier, List unprocessedGroupAnnotations) { + public EntityLog mergeEntityLog(ManualRedactions unprocessedManualRedactions, + EntityLog entityLog, + DossierEntity dossier, + List unprocessedGroupAnnotations) { final int analysisNumber = entityLog.getAnalysisNumber(); @@ -302,6 +305,9 @@ public class EntityLogMergeService { return mergeRecategorization(manualRecategorization, entityLogEntry, dossier, analysisNumber); } else if (localChange instanceof ManualForceRedaction manualForceRedaction) { mergeForceRedaction(manualForceRedaction, entityLogEntry, analysisNumber); + if (manualForceRedaction.getBasedOnDictAnnotationId() != null) { + trackLocalChangesBasedOnDictEntriesMap.put(manualForceRedaction.getAnnotationId(), manualForceRedaction.getBasedOnDictAnnotationId()); + } return null; } else { return null; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java index 070b46ce0..b2bf7a872 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java @@ -137,6 +137,7 @@ public class ManualRedactionMapper { for (ForceRedactionRequestModel forceRedactionRequestModel : forceRedactionRequests) { + String uuid = UUID.randomUUID().toString(); EntityLogEntry entityLogEntry = entityLogMongoWrapperService.getEntityLogEntryById(dossierId, fileId, forceRedactionRequestModel.getAnnotationId()); ForceRedactionRequest request = ForceRedactionRequest.builder() .annotationId(forceRedactionRequestModel.getAnnotationId()) @@ -146,7 +147,10 @@ public class ManualRedactionMapper { .build(); if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !entityLogEntry.getEngines().contains(Engine.IMPORTED) && entryIsEntityType(entityLogEntry)) { + request.setBasedOnDictAnnotationId(forceRedactionRequestModel.getAnnotationId()); + entityLogEntry.setId(uuid); manualRedactionEntryConsumer.accept(entityLogEntry); + request.setAnnotationId(uuid); } requests.add(RequestEntryPair.builder().request(request).entityLogEntry(entityLogEntry).build()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index d79cf7603..4df93d9d7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -228,4 +228,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/137-add-area-position-table.yaml - include: - file: db/changelog/tenant/138-add-page-range-table.yaml \ No newline at end of file + file: db/changelog/tenant/138-add-page-range-table.yaml + - include: + file: db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml new file mode 100644 index 000000000..dfca5160e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml @@ -0,0 +1,11 @@ +databaseChangeLog: + - changeSet: + id: add-annotation-info-to-manual-force + author: colariu + changes: + - addColumn: + columns: + - column: + name: based_on_dict_annotation_id + type: VARCHAR(255) + tableName: manual_force_redaction diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index bb41437c6..76eddafa1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1735,21 +1735,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); entityLogMongoService.upsertEntityLog(file.getDossierId(), file.getFileId(), entityLog); - manualRedactionClient.forceRedactionBulk(dossier.getId(), + ManualRedactionResponse forceResponse = manualRedactionClient.forceRedactionBulk(dossier.getId(), file.getId(), Set.of(ForceRedactionRequestModel.builder().annotationId("forceRedactionAnnotation").comment("comment").legalBasis("1").build())); var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true); assertEquals(allManualRedactions.getForceRedactions().size(), 1); - assertTrue(allManualRedactions.getForceRedactions() + assertFalse(allManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation"))); + assertTrue(allManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse.getManualAddResponses().get(0).getAnnotationId()))); var unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); assertEquals(unprocessedManualRedactions.getForceRedactions().size(), 1); - assertTrue(unprocessedManualRedactions.getForceRedactions() + assertFalse(unprocessedManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation"))); + assertTrue(allManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse.getManualAddResponses().get(0).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), @@ -2751,13 +2757,19 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .build())).getManualAddResponses() .get(0); - assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); + assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId()); assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis"); assertEquals(response.getEntityLogEntry().getState(), EntryState.APPLIED); assertEquals(response.getEntityLogEntry().getManualChanges() .get(0).getManualRedactionType(), ManualRedactionType.FORCE); assertNull(response.getEntityLogEntry().getManualChanges() .get(0).getProcessedDate()); + + var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true); + assertEquals(allManualRedactions.getForceRedactions().size(), 1); + assertTrue(allManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals(response.getAnnotationId()))); } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ForceRedactionRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ForceRedactionRequest.java index 2e73c7741..7ad6de92d 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ForceRedactionRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ForceRedactionRequest.java @@ -16,5 +16,6 @@ public class ForceRedactionRequest { private String legalBasis; private String comment; private int page; + private String basedOnDictAnnotationId; } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualForceRedaction.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualForceRedaction.java index eb3c5b701..1be3b9b42 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualForceRedaction.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualForceRedaction.java @@ -14,6 +14,7 @@ import lombok.experimental.SuperBuilder; public class ManualForceRedaction extends BaseAnnotation { private String legalBasis; + private String basedOnDictAnnotationId; @Override From 84ebccfe7915df2157cd35c63dd6fb47dfd25ff1 Mon Sep 17 00:00:00 2001 From: corinaolariu Date: Wed, 28 Aug 2024 13:45:06 +0300 Subject: [PATCH 2/5] RED-9495 - Remove here for locally resized dictionary entry should remove the entry completly - update unit test --- .../tests/ManualRedactionTest.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 76eddafa1..838acaee9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1767,27 +1767,39 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .dossierId(dossier.getId()) .build()); - manualRedactionClient.forceRedactionBulk(dossier.getId(), + ManualRedactionResponse forceResponse2 = manualRedactionClient.forceRedactionBulk(dossier.getId(), file.getId(), Set.of(ForceRedactionRequestModel.builder().annotationId("forceRedactionAnnotation2").comment("comment").legalBasis("1").build())); allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true); assertEquals(allManualRedactions.getForceRedactions().size(), 2); - assertTrue(allManualRedactions.getForceRedactions() + assertFalse(allManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation"))); assertTrue(allManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse.getManualAddResponses().get(0).getAnnotationId()))); + assertFalse(allManualRedactions.getForceRedactions() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation"))); + .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation2"))); + assertTrue(allManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse2.getManualAddResponses().get(1).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); assertEquals(unprocessedManualRedactions.getForceRedactions().size(), 1); - assertTrue(unprocessedManualRedactions.getForceRedactions() + assertFalse(unprocessedManualRedactions.getForceRedactions() .stream() .noneMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation"))); assertTrue(unprocessedManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse.getManualAddResponses().get(0).getAnnotationId()))); + assertFalse(unprocessedManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation2"))); + assertTrue(unprocessedManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse2.getManualAddResponses().get(1).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), From a8af78c501a32aa3abda0da753e5f001897a8659 Mon Sep 17 00:00:00 2001 From: corinaolariu Date: Wed, 28 Aug 2024 14:53:41 +0300 Subject: [PATCH 3/5] RED-9495 - Remove here for locally resized dictionary entry should remove the entry completly - update unit test --- .../v1/server/integration/tests/ManualRedactionTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 838acaee9..ba297d647 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1784,7 +1784,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation2"))); assertTrue(allManualRedactions.getForceRedactions() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse2.getManualAddResponses().get(1).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse2.getManualAddResponses().get(0).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); assertEquals(unprocessedManualRedactions.getForceRedactions().size(), 1); @@ -1799,7 +1799,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation2"))); assertTrue(unprocessedManualRedactions.getForceRedactions() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse2.getManualAddResponses().get(1).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse2.getManualAddResponses().get(0).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), From 5fdb7d37731c79dc053453655236b984f41569ea Mon Sep 17 00:00:00 2001 From: corinaolariu Date: Thu, 29 Aug 2024 09:34:26 +0300 Subject: [PATCH 4/5] RED-9548 - Actions results in wrong results for deadlocked annotation - fix unit test --- .../integration/tests/ManualRedactionTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index ba297d647..8b27659b9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1788,12 +1788,6 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); assertEquals(unprocessedManualRedactions.getForceRedactions().size(), 1); - assertFalse(unprocessedManualRedactions.getForceRedactions() - .stream() - .noneMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation"))); - assertTrue(unprocessedManualRedactions.getForceRedactions() - .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse.getManualAddResponses().get(0).getAnnotationId()))); assertFalse(unprocessedManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation2"))); @@ -1813,12 +1807,18 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true); assertEquals(allManualRedactions.getForceRedactions().size(), 2); + assertFalse(allManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation"))); assertTrue(allManualRedactions.getForceRedactions() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation"))); + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse.getManualAddResponses().get(0).getAnnotationId()))); + assertFalse(allManualRedactions.getForceRedactions() + .stream() + .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation2"))); assertTrue(allManualRedactions.getForceRedactions() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals("forceRedactionAnnotation2"))); + .anyMatch(entry -> entry.getAnnotationId().equals(forceResponse2.getManualAddResponses().get(0).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); assertTrue(unprocessedManualRedactions.getForceRedactions().isEmpty()); From a42e24d7f47387c77a318abeac270f98f9a117fa Mon Sep 17 00:00:00 2001 From: corinaolariu Date: Thu, 29 Aug 2024 11:43:30 +0300 Subject: [PATCH 5/5] RED-9495 - Remove here for locally resized dictionary entry should remove the entry completly - update db.changelog --- .../src/main/resources/db/changelog/db.changelog-tenant.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index f4f0c2f71..ce6383695 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -219,5 +219,5 @@ databaseChangeLog: file: db/changelog/tenant/132-add-based-on-dict-annotation-id-to-manual_changes.yaml - include: file: db/changelog/tenant/133-add-technical-name-to-legal_basis.yaml - - include: + - include: file: db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml \ No newline at end of file