Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7708-kilian' into 'master'

Browse Source 
RED-7708: multiple entries with same id in redactionLog/entityLog

Closes RED-7708

See merge request redactmanager/redaction-service!154
This commit is contained in:
Kilian Schüttler 2023-10-10 14:06:23 +02:00
commit cb82f90590
4 changed files with 33 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -136,18 +136,20 @@ public class EntityLogCreatorService {
private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) { private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
List<EntityLogEntry> entries = new ArrayList<>(); List<EntityLogEntry> entries = new ArrayList<>();
Set<String> processedIds = new HashSet<>();
document.getEntities() document.getEntities()
.stream().filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation).filter(entity -> !entity.removed()) .stream()
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));
document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId))); notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId)));
return entries; return entries;
} }
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId) { private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
Set<String> processedIds = new HashSet<>();
List<EntityLogEntry> redactionLogEntities = new ArrayList<>(); List<EntityLogEntry> redactionLogEntities = new ArrayList<>();
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
@ -178,7 +180,8 @@ public class EntityLogCreatorService {
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return EntityLogEntry.builder() return EntityLogEntry.builder()
.id(image.getId()) .id(image.getId())
.value(image.value()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint)) .value(image.value())
.color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
.value(image.value()) .value(image.value())
.type(imageType) .type(imageType)
.reason(image.buildReasonWithManualChangeDescriptions()) .reason(image.buildReasonWithManualChangeDescriptions())
@ -191,7 +194,8 @@ public class EntityLogCreatorService {
.section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
.imageHasTransparency(image.isTransparent()) .imageHasTransparency(image.isTransparent())
.manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(image)).entryType(buildEntryType(image)) .state(buildEntryState(image))
.entryType(buildEntryType(image))
.build(); .build();
} }
@ -201,13 +205,10 @@ public class EntityLogCreatorService {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(manualEntity.getEntityType()); boolean isHint = isHint(manualEntity.getEntityType());
return EntityLogEntry.builder() return EntityLogEntry.builder().id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
.id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
.reason(manualEntity.buildReasonWithManualChangeDescriptions()) .reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis()) .legalBasis(manualEntity.legalBasis())
.value(manualEntity.value()) .value(manualEntity.value()).type(type).state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity))
.type(type)
.state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.containingNodeId(Collections.emptyList()) .containingNodeId(Collections.emptyList())
.closestHeadline("") .closestHeadline("")
@ -231,7 +232,8 @@ public class EntityLogCreatorService {
Set<String> referenceIds = new HashSet<>(); Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
boolean isHint = isHint(entity.getEntityType()); boolean isHint = isHint(entity.getEntityType());
return EntityLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) return EntityLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
.reason(entity.buildReasonWithManualChangeDescriptions()) .reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis()) .legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@ -249,7 +251,8 @@ public class EntityLogCreatorService {
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
.reference(referenceIds) .reference(referenceIds)
.manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(entity)).entryType(buildEntryType(entity)) .state(buildEntryState(entity))
.entryType(buildEntryType(entity))
.build(); .build();
} }

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java
View File

@ -50,10 +50,11 @@ public class RedactionLogCreatorService {
Map<String, List<Comment>> comments) { Map<String, List<Comment>> comments) {
List<RedactionLogEntry> entries = new ArrayList<>(); List<RedactionLogEntry> entries = new ArrayList<>();
Set<String> processIds = new HashSet<>();
document.getEntities() document.getEntities()
.stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation) .stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(IEntity::active) .filter(IEntity::active)
.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments))); .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments, processIds)));
document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments))); document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments))); notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments)));
return entries; return entries;
@ -66,9 +67,8 @@ public class RedactionLogCreatorService {
} }
private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map<String, List<Comment>> comments) { private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map<String, List<Comment>> comments, Set<String> processedIds) {
Set<String> processedIds = new HashSet<>();
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>(); List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {

34
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl
View File

@ -144,6 +144,7 @@ rule "TestGuideline.0.0: create OECD number and year guideline mappings"
rule "TestGuideline.0.1: match OECD number and year with guideline mappings" rule "TestGuideline.0.1: match OECD number and year with guideline mappings"
salience 1 salience 1
when when
not Component(name == "Test_Guidelines_1")
GuidelineMapping($year: year, $number: number, $guideline: guideline) GuidelineMapping($year: year, $number: number, $guideline: guideline)
$guidelineNumber: Entity(type == "oecd_guideline_number", value == $number) $guidelineNumber: Entity(type == "oecd_guideline_number", value == $number)
$guidelineYear: Entity(type == "oecd_guideline_year", value == $year) $guidelineYear: Entity(type == "oecd_guideline_year", value == $year)
@ -175,7 +176,7 @@ rule "TestGuideline.2.0: All values of EPA guideline and EC guidelines"
rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy" rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy"
when when
$startDates: List(!isEmpty()) from collect (Entity(type == "experimental_start_date")) $startDates: List() from collect (Entity(type == "experimental_start_date"))
then then
componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates); componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates);
end end
@ -183,7 +184,7 @@ rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy"
rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy" rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy"
when when
$endDates: List(!isEmpty()) from collect (Entity(type == "experimental_end_date")) $endDates: List() from collect (Entity(type == "experimental_end_date"))
then then
componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates); componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates);
end end
@ -191,7 +192,7 @@ rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy"
rule "AnalysisCertificate.0.0: Unique values of certificate of analysis batch identification" rule "AnalysisCertificate.0.0: Unique values of certificate of analysis batch identification"
when when
$batchNumbers: List(!isEmpty()) from collect (Entity(type == "batch_number")) $batchNumbers: List() from collect (Entity(type == "batch_number"))
then then
componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Certificate_of_Analysis_Batch_Identification", $batchNumbers); componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Certificate_of_Analysis_Batch_Identification", $batchNumbers);
end end
@ -211,7 +212,7 @@ rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
FileAttribute(label == "OECD Number", value == $oecdNumber) FileAttribute(label == "OECD Number", value == $oecdNumber)
$guidelineDeviations: List() from collect (Entity(type == "guideline_deviation")) $guidelineDeviations: List() from collect (Entity(type == "guideline_deviation"))
then then
componentCreationService.asSentences("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations); componentCreationService.joining("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations, "\n");
end end
rule "Species.0.0: First found species" rule "Species.0.0: First found species"
@ -285,12 +286,12 @@ rule "Necropsy.0.0: Necropsy findings from longest section"
componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies, " "); componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies, " ");
end end
rule "Necropsy.0.1: Necropsy findings from longest section" rule "Necropsy.0.1: Necropsy findings joined with \n"
when when
FileAttribute(label == "OECD Number", value == "403" || value == "436") FileAttribute(label == "OECD Number", value == "403" || value == "436")
$necropsies: List() from collect (Entity(type == "necropsy_findings")) $necropsies: List() from collect (Entity(type == "necropsy_findings"))
then then
componentCreationService.asSentences("Necropsy.0.0", "Necropsy_Findings", $necropsies); componentCreationService.joining("Necropsy.0.0", "Necropsy_Findings", $necropsies, "\n");
end end
rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block" rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block"
@ -301,16 +302,7 @@ rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block"
componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " "); componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " ");
end end
rule "Necropsy.2.0: Necropsy findings as one block" rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block"
when
$oecdNumber: String() from List.of("403", "436")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$necropsies: List() from collect (Entity(type == "necropsy_findings"))
then
componentCreationService.joining("Necropsy.2.0", "Necropsy_Findings", $necropsies, " ");
end
rule "Necropsy.3.0: Conducted with 4 hours of exposure as one block"
when when
$oecdNumber: String() from List.of("403", "436") $oecdNumber: String() from List.of("403", "436")
FileAttribute(label == "OECD Number", value == $oecdNumber) FileAttribute(label == "OECD Number", value == $oecdNumber)
@ -342,7 +334,7 @@ rule "WeightBehavior.0.0: Weight change behavior as sentences"
FileAttribute(label == "OECD Number", value == "402") FileAttribute(label == "OECD Number", value == "402")
$weightChanges: List() from collect (Entity(type == "weight_behavior_changes")) $weightChanges: List() from collect (Entity(type == "weight_behavior_changes"))
then then
componentCreationService.asSentences("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges); componentCreationService.joining("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges, "\n");
end end
rule "MortalityStatement.0.0: Mortality statements as one block" rule "MortalityStatement.0.0: Mortality statements as one block"
@ -358,7 +350,7 @@ rule "ClinicalObservations.0.0: Clinical observations as sentences"
FileAttribute(label == "OECD Number", value == "403") FileAttribute(label == "OECD Number", value == "403")
$observations: List() from collect (Entity(type == "clinical_observations")) $observations: List() from collect (Entity(type == "clinical_observations"))
then then
componentCreationService.asSentences("MortalityStatement.0.0", "Clinical_Observations", $observations); componentCreationService.joining("MortalityStatement.0.0", "Clinical_Observations", $observations, "\n");
end end
rule "BodyWeight.0.0: Bodyweight changes as sentences" rule "BodyWeight.0.0: Bodyweight changes as sentences"
@ -366,7 +358,7 @@ rule "BodyWeight.0.0: Bodyweight changes as sentences"
FileAttribute(label == "OECD Number", value == "403") FileAttribute(label == "OECD Number", value == "403")
$weightChanges: List() from collect (Entity(type == "bodyweight_changes")) $weightChanges: List() from collect (Entity(type == "bodyweight_changes"))
then then
componentCreationService.asSentences("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges); componentCreationService.joining("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges, "\n");
end end
rule "Detailing.0.0: Detailing of reported changes as one block" rule "Detailing.0.0: Detailing of reported changes as one block"
@ -421,7 +413,7 @@ rule "ClinicalSigns.0.0: Clinical signs as sentences"
FileAttribute(label == "OECD Number", value == $oecdNumber) FileAttribute(label == "OECD Number", value == $oecdNumber)
$clinicalSigns: List() from collect (Entity(type == "clinical_signs")) $clinicalSigns: List() from collect (Entity(type == "clinical_signs"))
then then
componentCreationService.asSentences("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns); componentCreationService.joining("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns, "\n");
end end
rule "DoseMortality.0.0: Dose mortality joined with dose from same table row" rule "DoseMortality.0.0: Dose mortality joined with dose from same table row"
@ -457,7 +449,7 @@ rule "PrelimResults.0.0: Preliminary test results as sentences"
FileAttribute(label == "OECD Number", value == $oecdNumber) FileAttribute(label == "OECD Number", value == $oecdNumber)
$results: List() from collect (Entity(type == "preliminary_test_results")) $results: List() from collect (Entity(type == "preliminary_test_results"))
then then
componentCreationService.asSentences("PrelimResults.0.0", "Preliminary_Test_Results", $results); componentCreationService.joining("PrelimResults.0.0", "Preliminary_Test_Results", $results, "\n");
end end
rule "TestResults.0.0: Test results as one block" rule "TestResults.0.0: Test results as one block"

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1097,7 +1097,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6 // Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY" rule "X.6.0: remove Entity of lower rank, when contained by entity of type ENTITY"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())