From 492bb56513c1c98c42ff608b12c21e51f5a54f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Tue, 10 Oct 2023 14:06:23 +0200 Subject: [PATCH] RED-7708: multiple entries with same id in redactionLog/entityLog --- .../service/EntityLogCreatorService.java | 29 +++++++++------- .../service/RedactionLogCreatorService.java | 6 ++-- .../drools/documine_flora_components.drl | 34 +++++++------------ .../src/test/resources/drools/rules.drl | 2 +- 4 files changed, 33 insertions(+), 38 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index cb517347..e915cc0f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -136,18 +136,20 @@ public class EntityLogCreatorService { private List createEntityLogEntries(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { List entries = new ArrayList<>(); + Set processedIds = new HashSet<>(); document.getEntities() - .stream().filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation).filter(entity -> !entity.removed()) - .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); + .stream() + .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation) + .filter(entity -> !entity.removed()) + .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds))); document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId))); return entries; } - private List toEntityLogEntries(TextEntity textEntity, String dossierTemplateId) { + private List toEntityLogEntries(TextEntity textEntity, String dossierTemplateId, Set processedIds) { - Set processedIds = new HashSet<>(); List redactionLogEntities = new ArrayList<>(); for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { @@ -178,7 +180,8 @@ public class EntityLogCreatorService { boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); return EntityLogEntry.builder() .id(image.getId()) - .value(image.value()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint)) + .value(image.value()) + .color(getColor(imageType, dossierTemplateId, image.applied(), isHint)) .value(image.value()) .type(imageType) .reason(image.buildReasonWithManualChangeDescriptions()) @@ -191,7 +194,8 @@ public class EntityLogCreatorService { .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) .imageHasTransparency(image.isTransparent()) .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) - .state(buildEntryState(image)).entryType(buildEntryType(image)) + .state(buildEntryState(image)) + .entryType(buildEntryType(image)) .build(); } @@ -201,13 +205,10 @@ public class EntityLogCreatorService { String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); boolean isHint = isHint(manualEntity.getEntityType()); - return EntityLogEntry.builder() - .id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint)) + return EntityLogEntry.builder().id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint)) .reason(manualEntity.buildReasonWithManualChangeDescriptions()) .legalBasis(manualEntity.legalBasis()) - .value(manualEntity.value()) - .type(type) - .state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity)) + .value(manualEntity.value()).type(type).state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity)) .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) .containingNodeId(Collections.emptyList()) .closestHeadline("") @@ -231,7 +232,8 @@ public class EntityLogCreatorService { Set referenceIds = new HashSet<>(); entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); boolean isHint = isHint(entity.getEntityType()); - return EntityLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) + return EntityLogEntry.builder() + .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) @@ -249,7 +251,8 @@ public class EntityLogCreatorService { .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) .reference(referenceIds) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) - .state(buildEntryState(entity)).entryType(buildEntryType(entity)) + .state(buildEntryState(entity)) + .entryType(buildEntryType(entity)) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index 7c98a09d..79340cd5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -50,10 +50,11 @@ public class RedactionLogCreatorService { Map> comments) { List entries = new ArrayList<>(); + Set processIds = new HashSet<>(); document.getEntities() .stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation) .filter(IEntity::active) - .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments))); + .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments, processIds))); document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments))); notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments))); return entries; @@ -66,9 +67,8 @@ public class RedactionLogCreatorService { } - private List toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map> comments) { + private List toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map> comments, Set processedIds) { - Set processedIds = new HashSet<>(); List redactionLogEntities = new ArrayList<>(); for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl index 8080ea62..f1d18f26 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl @@ -144,6 +144,7 @@ rule "TestGuideline.0.0: create OECD number and year guideline mappings" rule "TestGuideline.0.1: match OECD number and year with guideline mappings" salience 1 when + not Component(name == "Test_Guidelines_1") GuidelineMapping($year: year, $number: number, $guideline: guideline) $guidelineNumber: Entity(type == "oecd_guideline_number", value == $number) $guidelineYear: Entity(type == "oecd_guideline_year", value == $year) @@ -175,7 +176,7 @@ rule "TestGuideline.2.0: All values of EPA guideline and EC guidelines" rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy" when - $startDates: List(!isEmpty()) from collect (Entity(type == "experimental_start_date")) + $startDates: List() from collect (Entity(type == "experimental_start_date")) then componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates); end @@ -183,7 +184,7 @@ rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy" rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy" when - $endDates: List(!isEmpty()) from collect (Entity(type == "experimental_end_date")) + $endDates: List() from collect (Entity(type == "experimental_end_date")) then componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates); end @@ -191,7 +192,7 @@ rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy" rule "AnalysisCertificate.0.0: Unique values of certificate of analysis batch identification" when - $batchNumbers: List(!isEmpty()) from collect (Entity(type == "batch_number")) + $batchNumbers: List() from collect (Entity(type == "batch_number")) then componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Certificate_of_Analysis_Batch_Identification", $batchNumbers); end @@ -211,7 +212,7 @@ rule "GuidelineDeviation.0.0: Guideline deviation as sentences" FileAttribute(label == "OECD Number", value == $oecdNumber) $guidelineDeviations: List() from collect (Entity(type == "guideline_deviation")) then - componentCreationService.asSentences("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations); + componentCreationService.joining("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations, "\n"); end rule "Species.0.0: First found species" @@ -285,12 +286,12 @@ rule "Necropsy.0.0: Necropsy findings from longest section" componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies, " "); end -rule "Necropsy.0.1: Necropsy findings from longest section" +rule "Necropsy.0.1: Necropsy findings joined with \n" when FileAttribute(label == "OECD Number", value == "403" || value == "436") $necropsies: List() from collect (Entity(type == "necropsy_findings")) then - componentCreationService.asSentences("Necropsy.0.0", "Necropsy_Findings", $necropsies); + componentCreationService.joining("Necropsy.0.0", "Necropsy_Findings", $necropsies, "\n"); end rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block" @@ -301,16 +302,7 @@ rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block" componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " "); end -rule "Necropsy.2.0: Necropsy findings as one block" - when - $oecdNumber: String() from List.of("403", "436") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $necropsies: List() from collect (Entity(type == "necropsy_findings")) - then - componentCreationService.joining("Necropsy.2.0", "Necropsy_Findings", $necropsies, " "); - end - -rule "Necropsy.3.0: Conducted with 4 hours of exposure as one block" +rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block" when $oecdNumber: String() from List.of("403", "436") FileAttribute(label == "OECD Number", value == $oecdNumber) @@ -342,7 +334,7 @@ rule "WeightBehavior.0.0: Weight change behavior as sentences" FileAttribute(label == "OECD Number", value == "402") $weightChanges: List() from collect (Entity(type == "weight_behavior_changes")) then - componentCreationService.asSentences("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges); + componentCreationService.joining("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges, "\n"); end rule "MortalityStatement.0.0: Mortality statements as one block" @@ -358,7 +350,7 @@ rule "ClinicalObservations.0.0: Clinical observations as sentences" FileAttribute(label == "OECD Number", value == "403") $observations: List() from collect (Entity(type == "clinical_observations")) then - componentCreationService.asSentences("MortalityStatement.0.0", "Clinical_Observations", $observations); + componentCreationService.joining("MortalityStatement.0.0", "Clinical_Observations", $observations, "\n"); end rule "BodyWeight.0.0: Bodyweight changes as sentences" @@ -366,7 +358,7 @@ rule "BodyWeight.0.0: Bodyweight changes as sentences" FileAttribute(label == "OECD Number", value == "403") $weightChanges: List() from collect (Entity(type == "bodyweight_changes")) then - componentCreationService.asSentences("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges); + componentCreationService.joining("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges, "\n"); end rule "Detailing.0.0: Detailing of reported changes as one block" @@ -421,7 +413,7 @@ rule "ClinicalSigns.0.0: Clinical signs as sentences" FileAttribute(label == "OECD Number", value == $oecdNumber) $clinicalSigns: List() from collect (Entity(type == "clinical_signs")) then - componentCreationService.asSentences("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns); + componentCreationService.joining("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns, "\n"); end rule "DoseMortality.0.0: Dose mortality joined with dose from same table row" @@ -457,7 +449,7 @@ rule "PrelimResults.0.0: Preliminary test results as sentences" FileAttribute(label == "OECD Number", value == $oecdNumber) $results: List() from collect (Entity(type == "preliminary_test_results")) then - componentCreationService.asSentences("PrelimResults.0.0", "Preliminary_Test_Results", $results); + componentCreationService.joining("PrelimResults.0.0", "Preliminary_Test_Results", $results, "\n"); end rule "TestResults.0.0: Test results as one block" diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 7c8b7beb..085028cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1097,7 +1097,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" // Rule unit: X.6 -rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY" +rule "X.6.0: remove Entity of lower rank, when contained by entity of type ENTITY" salience 32 when $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())