From ebb16c9110666fe477852f03b63a30b4b0dd061e Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Mon, 18 Sep 2023 14:17:20 +0200 Subject: [PATCH] DM-428 - Fix bulk annotations not saving correctly --- .../service/EntityLogCreatorService.java | 1 + .../ManualChangesApplicationService.java | 4 +- .../service/RedactionLogCreatorService.java | 6 ++- .../document/EntityCreationService.java | 8 +++- .../v1/server/RedactionAcceptanceTest.java | 9 ++-- .../ManualChangesIntegrationTest.java | 46 ------------------- 6 files changed, 19 insertions(+), 55 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 602c758a..61c04585 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -146,6 +146,7 @@ public class EntityLogCreatorService { return EntityLogEntry.builder() .id(image.getId()).value(image.value()) .color(getColor(imageType, dossierTemplateId, image.applied())) + .value(image.value()) .type(imageType) .reason(image.buildReasonWithManualChangeDescriptions()) .legalBasis(image.legalBasis()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index e89b7489..336c4092 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -32,10 +32,10 @@ public class ManualChangesApplicationService { } // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed! if (IEntityToBeReCategorized instanceof TextEntity textEntity) { - TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode()); + TextEntity recategorizedEntity = entityCreationService.copyEntityWithoutRules(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode()); recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage()); recategorizedEntity.getManualOverwrite().addChange(manualRecategorization); - textEntity.removeFromGraph(); + textEntity.remove("FINAL.0.0", "removed by manual recategorization"); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index 589bfe0a..55773f59 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -23,6 +23,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; @@ -52,7 +53,7 @@ public class RedactionLogCreatorService { document.getEntities() .stream() .filter(RedactionLogCreatorService::isEntityOrRecommendationType) - .filter(entity -> !entity.removed()) + .filter(IEntity::active) .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments))); document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments))); notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments))); @@ -185,7 +186,7 @@ public class RedactionLogCreatorService { .color(getColor(type, dossierTemplateId, manualEntity.applied())) .reason(manualEntity.buildReasonWithManualChangeDescriptions()) .legalBasis(manualEntity.legalBasis()) - .value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue())) + .value(manualEntity.value()) .type(type) .redacted(manualEntity.applied()) .isHint(isHint) @@ -250,6 +251,7 @@ public class RedactionLogCreatorService { .id(image.getId()) .color(getColor(imageType, dossierTemplateId, image.applied())) .isImage(true) + .value(image.value()) .type(imageType) .redacted(image.applied()) .reason(image.buildReasonWithManualChangeDescriptions()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index ce4c1270..0e1c21c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -576,9 +576,15 @@ public class EntityCreationService { public TextEntity copyEntity(TextEntity entity, String type, EntityType entityType, SemanticNode node) { + var newEntity = copyEntityWithoutRules(entity, type, entityType, node); + newEntity.addMatchedRules(entity.getMatchedRuleList()); + return newEntity; + } + + public TextEntity copyEntityWithoutRules(TextEntity entity, String type, EntityType entityType, SemanticNode node) { + TextEntity newEntity = TextEntity.initialEntityNode(entity.getTextRange(), type, entityType); newEntity.addEngines(entity.getEngines()); - newEntity.addMatchedRules(entity.getMatchedRuleList()); newEntity.getManualOverwrite().addChanges(entity.getManualOverwrite().getManualChangeLog()); newEntity.setDictionaryEntry(entity.isDictionaryEntry()); newEntity.setDossierDictionaryEntry(entity.isDossierDictionaryEntry()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 585204a3..8dc59459 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -1,7 +1,7 @@ package com.iqser.red.service.redaction.v1.server; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; +import static org.wildfly.common.Assert.assertFalse; import static org.wildfly.common.Assert.assertTrue; import java.io.FileOutputStream; @@ -33,12 +33,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; -import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; @@ -180,8 +181,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { System.out.println("Finished reanalysis"); var redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertTrue(findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> !entry.isRecommendation()).noneMatch(RedactionLogEntry::isRedacted)); - + assertTrue(findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> !entry.isRecommendation()) + .findFirst().get().getChanges().get(1).getType().equals(ChangeType.REMOVED)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index 629fa5c8..b2ad45db 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -249,52 +249,6 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { assertFalse(entity.removed()); } - - @Test - public void testManualRecategorizeRoundTrip() { - - Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); - - TextEntity entity = entities.stream() - .min(Comparator.comparingInt(textEntity -> textEntity.getTextRange().start())) - .orElseThrow(() -> new NotFoundException("no entity found!")); - String originalType = entity.getType(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); - - OffsetDateTime start = OffsetDateTime.now(); - ManualRecategorization recategorization = ManualRecategorization.builder() - .type("any other type") - .annotationId(initialId) - .status(AnnotationStatus.APPROVED) - .requestDate(start) - .build(); - ManualRecategorization recategorization2 = ManualRecategorization.builder() - .type(originalType) - .annotationId(initialId) - .status(AnnotationStatus.APPROVED) - .requestDate(start.plusSeconds(1)) - .build(); - - - kieSession.insert(recategorization); - kieSession.insert(recategorization2); - kieSession.insert(document); - document.streamAllSubNodes().forEach(kieSession::insert); - kieSession.fireAllRules(); - kieSession.dispose(); - - var recategorizedEntity = document.getEntities() - .stream() - .filter(entity2 -> entity2.matchesAnnotationId(initialId)) - .findFirst() - .orElseThrow(() -> new NotFoundException("recategorized entity not found!")); - - assertEquals(originalType, recategorizedEntity.getManualOverwrite().getType().orElse(recategorizedEntity.getType())); - - } - - private void assertRectanglesAlmostEqual(Collection rects1, Collection rects2) { if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {