From a15e6c2651a86fa33695144f0f875a04c68180c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Fri, 22 Sep 2023 14:28:36 +0200 Subject: [PATCH] DM-285: adjust ComponentLog to new Format --- .../build.gradle.kts | 2 +- .../v1/server/model/component/Component.java | 4 +- .../v1/server/service/AnalyzeService.java | 11 +- .../service/ComponentLogCreatorService.java | 37 ++--- .../document/ComponentCreationService.java | 150 +++++++++--------- .../drools/documine_flora_components.drl | 4 +- 6 files changed, 100 insertions(+), 108 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 278e3f62..fcf8b9e5 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.70.0" val jacksonVersion = "2.15.2" val droolsVersion = "8.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.181.0" +val persistenceServiceVersion = "2.182.0" configurations { all { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java index d820e712..7f90fa7d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java @@ -20,9 +20,9 @@ import lombok.experimental.FieldDefaults; public class Component { RuleIdentifier matchedRule; - String category; + String name; String value; - String transformation; + String valueDescription; List references; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 6c63a1ac..bc6b3d46 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -197,7 +197,11 @@ public class AnalyzeService { RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, - document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds, dictionary.getVersion()); + document, + notFoundManualRedactionEntries, + previousEntityLog, + sectionsToReanalyseIds, + dictionary.getVersion()); return finalizeAnalysis(analyzeRequest, startTime, @@ -294,10 +298,7 @@ public class AnalyzeService { addedFileAttributes.stream().toList()); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), - components, - dictionaryVersion, - kieWrapperComponentRules.rulesVersion()); + ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java index 9db4087a..339c9156 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java @@ -5,53 +5,50 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentValue; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.EntityReference; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators; @Service public class ComponentLogCreatorService { - public ComponentLog buildComponentLog(int analysisNumber, List components, DictionaryVersion dictionaryVersion, long rulesVersion) { + public ComponentLog buildComponentLog(int analysisNumber, List components, long componentRulesVersion) { - List componentLogCategories = components.stream() - .collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList()))) + List componentLogComponents = components.stream() + .collect(Collectors.groupingBy(Component::getName, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList()))) .entrySet() .stream() - .map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue())) + .map(entry -> new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.Component(entry.getKey(), entry.getValue())) .toList(); - return new ComponentLog(analysisNumber, componentLogCategories, dictionaryVersion.getDossierVersion(), dictionaryVersion.getDossierTemplateVersion(), rulesVersion); + return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents); } - private ComponentLogEntry buildComponentLogEntry(Component component) { + private ComponentValue buildComponentLogEntry(Component component) { - return ComponentLogEntry.builder() - .value(component.getValue()).matchedRule(component.getMatchedRule().toString()) - .transformation(component.getTransformation()) - .componentEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList())) + return ComponentValue.builder() + .value(component.getValue()).originalValue(component.getValue()) + .componentRuleId(component.getMatchedRule().toString()) + .valueDescription(component.getValueDescription()) + .entityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList())) .build(); } - private List toComponentEntityReferences(List references) { + private List toComponentEntityReferences(List references) { return references.stream().map(this::toComponentEntityReference).toList(); } - private ComponentEntityReference toComponentEntityReference(Entity entity) { + private EntityReference toComponentEntityReference(Entity entity) { - return ComponentEntityReference.builder().id(entity.getId()).value(entity.getValue()) - .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)) - .reason(entity.getReason()) - .ruleIdentifier(entity.getMatchedRule()) + return EntityReference.builder().id(entity.getId()) + .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule()) .type(entity.getType()) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java index 8b64f02d..9fbe252d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java @@ -32,44 +32,50 @@ public class ComponentCreationService { Set referencedEntities = new HashSet<>(); - public void firstOrElse(String ruleIdentifier, String category, Collection entities, String fallback) { + public void firstOrElse(String ruleIdentifier, String name, Collection entities, String fallback) { - String transformation = String.format("First found value or else '%s'", fallback); + String valueDescription = String.format("First found value or else '%s'", fallback); String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback); - create(ruleIdentifier, category, value, transformation, entities); + create(ruleIdentifier, name, value, valueDescription, entities); } - public void joining(String ruleIdentifier, String category, Collection entities) { + public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection references) { - joining(ruleIdentifier, category, entities, ", "); + referencedEntities.addAll(references); + + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name) + .value(value).valueDescription(valueDescription) + .references(new LinkedList<>(references)) + .build()); } - public void joiningFromFirstSectionOnly(String ruleIdentifier, String category, Collection entities) { + public void joining(String ruleIdentifier, String name, Collection entities) { - joiningFromFirstSectionOnly(ruleIdentifier, category, entities, ", "); + joining(ruleIdentifier, name, entities, ", "); } - public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String category, Collection entities) { + public void joining(String ruleIdentifier, String name, Collection entities, String delimiter) { - joiningUniqueFromFirstSectionOnly(ruleIdentifier, category, entities, ", "); - } - - - public void joining(String ruleIdentifier, String category, Collection entities, String delimiter) { - - String transformation = String.format("Joining all values with '%s'", delimiter); + String valueDescription = String.format("Joining all values with '%s'", delimiter); String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter)); - create(ruleIdentifier, category, value, transformation, entities); + create(ruleIdentifier, name, value, valueDescription, entities); } - public void joiningFromFirstSectionOnly(String ruleIdentifier, String category, Collection entities, String delimiter) { + public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection entities) { + + joiningFromFirstSectionOnly(ruleIdentifier, name, entities, ", "); + } + + + public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection entities, String delimiter) { List entitiesFromFirstSection = findEntitiesFromFirstSection(entities); - joining(ruleIdentifier, category, entitiesFromFirstSection, delimiter); + joining(ruleIdentifier, name, entitiesFromFirstSection, delimiter); } @@ -84,17 +90,16 @@ public class ComponentCreationService { } - public void joiningFromLongestSectionOnly(String ruleIdentifier, String category, Collection entities, String delimiter) { + public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection entities) { - List entitiesFromLongestSection = findEntitiesFromLongestSection(entities); - joining(ruleIdentifier, category, entitiesFromLongestSection, delimiter); + joiningUniqueFromFirstSectionOnly(ruleIdentifier, name, entities, ", "); } - public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String category, Collection entities, String delimiter) { + public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection entities, String delimiter) { - List entitiesFromLongestSection = findEntitiesFromLongestSection(entities); - joiningUnique(ruleIdentifier, category, entitiesFromLongestSection, delimiter); + List entitiesFromFirstSection = findEntitiesFromFirstSection(entities); + joiningUnique(ruleIdentifier, name, entitiesFromFirstSection, delimiter); } @@ -121,28 +126,35 @@ public class ComponentCreationService { } - public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String category, Collection entities, String delimiter) { + public void joiningUnique(String ruleIdentifier, String name, Collection entities, String delimiter) { - List entitiesFromFirstSection = findEntitiesFromFirstSection(entities); - joiningUnique(ruleIdentifier, category, entitiesFromFirstSection, delimiter); - } - - - public void joiningUnique(String ruleIdentifier, String category, Collection entities) { - - joiningUnique(ruleIdentifier, category, entities, ", "); - } - - - public void joiningUnique(String ruleIdentifier, String category, Collection entities, String delimiter) { - - String transformation = String.format("Joining all values with '%s'", delimiter); + String valueDescription = String.format("Joining all values with '%s'", delimiter); String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter)); - create(ruleIdentifier, category, value, transformation, entities); + create(ruleIdentifier, name, value, valueDescription, entities); } - public void asSentences(String ruleIdentifier, String category, Collection entities) { + public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection entities, String delimiter) { + + List entitiesFromLongestSection = findEntitiesFromLongestSection(entities); + joining(ruleIdentifier, name, entitiesFromLongestSection, delimiter); + } + + + public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection entities, String delimiter) { + + List entitiesFromLongestSection = findEntitiesFromLongestSection(entities); + joiningUnique(ruleIdentifier, name, entitiesFromLongestSection, delimiter); + } + + + public void joiningUnique(String ruleIdentifier, String name, Collection entities) { + + joiningUnique(ruleIdentifier, name, entities, ", "); + } + + + public void asSentences(String ruleIdentifier, String name, Collection entities) { if (entities.isEmpty()) { return; @@ -153,23 +165,20 @@ public class ComponentCreationService { iterator.setText(entity.getValue()); int start = iterator.first(); for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) { - create(ruleIdentifier, category, entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), "Split into sentences", entity); + create(ruleIdentifier, name, entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), "Split into sentences", entity); } } } - public void convertDates(String ruleIdentifier, String category, Collection entities) { + public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) { - convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy"); - } - - - public void convertDates(String ruleIdentifier, String category, Collection entities, String resultFormat) { - - String transformation = "Convert values of type to dd/MM/yyyy joined with ', '"; - String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", ")); - create(ruleIdentifier, category, date, transformation, entities); + referencedEntities.add(reference); + List referenceList = new LinkedList<>(); + referenceList.add(reference); + kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription) + .references(referenceList) + .build()); } @@ -181,45 +190,30 @@ public class ComponentCreationService { } - public void create(String ruleIdentifier, String category, String value, String transformation, Collection references) { + public void convertDates(String ruleIdentifier, String name, Collection entities) { - referencedEntities.addAll(references); - - kieSession.insert(Component.builder() - .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) - .category(category) - .value(value).transformation(transformation).references(new LinkedList<>(references)).build()); + convertDates(ruleIdentifier, name, entities, "dd/MM/yyyy"); } - public void create(String ruleIdentifier, String category, String value, String transformation, Entity reference) { + public void convertDates(String ruleIdentifier, String name, Collection entities, String resultFormat) { - referencedEntities.add(reference); - List referenceList = new LinkedList<>(); - referenceList.add(reference); - kieSession.insert(Component.builder() - .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) - .category(category) - .value(value) - .transformation(transformation) - .references(referenceList) - .build()); + String valueDescription = "Convert values of type to dd/MM/yyyy joined with ', '"; + String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", ")); + create(ruleIdentifier, name, date, valueDescription, entities); } - public void create(String ruleIdentifier, String category, String value, String transformation) { + public void create(String ruleIdentifier, String name, String value, String valueDescription) { - create(ruleIdentifier, category, value, transformation, Collections.emptyList()); + create(ruleIdentifier, name, value, valueDescription, Collections.emptyList()); } - public void create(String ruleIdentifier, String category, String value) { + public void create(String ruleIdentifier, String name, String value) { - kieSession.insert(Component.builder() - .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) - .category(category) - .value(value) - .transformation("") + kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name) + .value(value).valueDescription("") .references(Collections.emptyList()) .build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl index e98d02fb..21eb2477 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl @@ -75,7 +75,7 @@ rule "PerformingLaboratory.2.0: Performing Laboratory" rule "PerformingLaboratory.0.2: Performing Laboratory" salience -1 when - not Component(category == "Performing_Laboratory") + not Component(name == "Performing_Laboratory") then componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback"); end @@ -227,7 +227,7 @@ rule "DefaultComponents.999.0: Create components for all unmapped entities." rule "X.0.0: merge duplicate component references" when $first: Component() - $duplicate: Component(this != $first, category == $first.category, value == $first.value) + $duplicate: Component(this != $first, name == $first.name, value == $first.value) then $first.getReferences().addAll($duplicate.getReferences()); retract($duplicate);