Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'DM-285' into 'master'

Browse Source 
DM-285: adjust ComponentLog to new Format

Closes DM-285

See merge request redactmanager/redaction-service!141
This commit is contained in:
Kilian Schüttler 2023-09-22 14:28:36 +02:00
commit ce580b6285
6 changed files with 100 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.70.0"
val jacksonVersion = "2.15.2" val jacksonVersion = "2.15.2"
val droolsVersion = "8.44.0.Final" val droolsVersion = "8.44.0.Final"
val pdfBoxVersion = "3.0.0" val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.181.0" val persistenceServiceVersion = "2.182.0"
configurations { configurations {
all { all {

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java
View File

@ -20,9 +20,9 @@ import lombok.experimental.FieldDefaults;
public class Component { public class Component {
RuleIdentifier matchedRule; RuleIdentifier matchedRule;
String category; String name;
String value; String value;
String transformation; String valueDescription;
List<Entity> references; List<Entity> references;

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -197,7 +197,11 @@ public class AnalyzeService {
RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds, dictionary.getVersion()); document,
notFoundManualRedactionEntries,
previousEntityLog,
sectionsToReanalyseIds,
dictionary.getVersion());
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
@ -294,10 +298,7 @@ public class AnalyzeService {
addedFileAttributes.stream().toList()); addedFileAttributes.stream().toList());
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
components,
dictionaryVersion,
kieWrapperComponentRules.rulesVersion());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);

37
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java
View File

@ -5,53 +5,50 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.EntityReference;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators; import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;
@Service @Service
public class ComponentLogCreatorService { public class ComponentLogCreatorService {
public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, DictionaryVersion dictionaryVersion, long rulesVersion) { public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
List<ComponentLogCategory> componentLogCategories = components.stream() List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.Component> componentLogComponents = components.stream()
.collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList()))) .collect(Collectors.groupingBy(Component::getName, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
.entrySet() .entrySet()
.stream() .stream()
.map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue())) .map(entry -> new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.Component(entry.getKey(), entry.getValue()))
.toList(); .toList();
return new ComponentLog(analysisNumber, componentLogCategories, dictionaryVersion.getDossierVersion(), dictionaryVersion.getDossierTemplateVersion(), rulesVersion); return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
} }
private ComponentLogEntry buildComponentLogEntry(Component component) { private ComponentValue buildComponentLogEntry(Component component) {
return ComponentLogEntry.builder() return ComponentValue.builder()
.value(component.getValue()).matchedRule(component.getMatchedRule().toString()) .value(component.getValue()).originalValue(component.getValue())
.transformation(component.getTransformation()) .componentRuleId(component.getMatchedRule().toString())
.componentEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList())) .valueDescription(component.getValueDescription())
.entityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList()))
.build(); .build();
} }
private List<ComponentEntityReference> toComponentEntityReferences(List<Entity> references) { private List<EntityReference> toComponentEntityReferences(List<Entity> references) {
return references.stream().map(this::toComponentEntityReference).toList(); return references.stream().map(this::toComponentEntityReference).toList();
} }
private ComponentEntityReference toComponentEntityReference(Entity entity) { private EntityReference toComponentEntityReference(Entity entity) {
return ComponentEntityReference.builder().id(entity.getId()).value(entity.getValue()) return EntityReference.builder().id(entity.getId())
.page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)) .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule())
.reason(entity.getReason())
.ruleIdentifier(entity.getMatchedRule())
.type(entity.getType()) .type(entity.getType())
.build(); .build();
} }

150
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java
View File

@ -32,44 +32,50 @@ public class ComponentCreationService {
Set<Entity> referencedEntities = new HashSet<>(); Set<Entity> referencedEntities = new HashSet<>();
public void firstOrElse(String ruleIdentifier, String category, Collection<Entity> entities, String fallback) { public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
String transformation = String.format("First found value or else '%s'", fallback); String valueDescription = String.format("First found value or else '%s'", fallback);
String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback); String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback);
create(ruleIdentifier, category, value, transformation, entities); create(ruleIdentifier, name, value, valueDescription, entities);
} }
public void joining(String ruleIdentifier, String category, Collection<Entity> entities) { public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection<Entity> references) {
joining(ruleIdentifier, category, entities, ", "); referencedEntities.addAll(references);
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
.value(value).valueDescription(valueDescription)
.references(new LinkedList<>(references))
.build());
} }
public void joiningFromFirstSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities) { public void joining(String ruleIdentifier, String name, Collection<Entity> entities) {
joiningFromFirstSectionOnly(ruleIdentifier, category, entities, ", "); joining(ruleIdentifier, name, entities, ", ");
} }
public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities) { public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
joiningUniqueFromFirstSectionOnly(ruleIdentifier, category, entities, ", "); String valueDescription = String.format("Joining all values with '%s'", delimiter);
}
public void joining(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
String transformation = String.format("Joining all values with '%s'", delimiter);
String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter)); String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
create(ruleIdentifier, category, value, transformation, entities); create(ruleIdentifier, name, value, valueDescription, entities);
} }
public void joiningFromFirstSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) { public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
joiningFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
}
public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities); List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
joining(ruleIdentifier, category, entitiesFromFirstSection, delimiter); joining(ruleIdentifier, name, entitiesFromFirstSection, delimiter);
} }
@ -84,17 +90,16 @@ public class ComponentCreationService {
} }
public void joiningFromLongestSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) { public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities); joiningUniqueFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
joining(ruleIdentifier, category, entitiesFromLongestSection, delimiter);
} }
public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) { public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities); List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
joiningUnique(ruleIdentifier, category, entitiesFromLongestSection, delimiter); joiningUnique(ruleIdentifier, name, entitiesFromFirstSection, delimiter);
} }
@ -121,28 +126,35 @@ public class ComponentCreationService {
} }
public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) { public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities); String valueDescription = String.format("Joining all values with '%s'", delimiter);
joiningUnique(ruleIdentifier, category, entitiesFromFirstSection, delimiter);
}
public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities) {
joiningUnique(ruleIdentifier, category, entities, ", ");
}
public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
String transformation = String.format("Joining all values with '%s'", delimiter);
String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter)); String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
create(ruleIdentifier, category, value, transformation, entities); create(ruleIdentifier, name, value, valueDescription, entities);
} }
public void asSentences(String ruleIdentifier, String category, Collection<Entity> entities) { public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
joining(ruleIdentifier, name, entitiesFromLongestSection, delimiter);
}
public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
joiningUnique(ruleIdentifier, name, entitiesFromLongestSection, delimiter);
}
public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities) {
joiningUnique(ruleIdentifier, name, entities, ", ");
}
public void asSentences(String ruleIdentifier, String name, Collection<Entity> entities) {
if (entities.isEmpty()) { if (entities.isEmpty()) {
return; return;
@ -153,23 +165,20 @@ public class ComponentCreationService {
iterator.setText(entity.getValue()); iterator.setText(entity.getValue());
int start = iterator.first(); int start = iterator.first();
for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) { for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
create(ruleIdentifier, category, entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), "Split into sentences", entity); create(ruleIdentifier, name, entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), "Split into sentences", entity);
} }
} }
} }
public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities) { public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) {
convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy"); referencedEntities.add(reference);
} List<Entity> referenceList = new LinkedList<>();
referenceList.add(reference);
kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities, String resultFormat) { .references(referenceList)
.build());
String transformation = "Convert values of type to dd/MM/yyyy joined with ', '";
String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
create(ruleIdentifier, category, date, transformation, entities);
} }
@ -181,45 +190,30 @@ public class ComponentCreationService {
} }
public void create(String ruleIdentifier, String category, String value, String transformation, Collection<Entity> references) { public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities) {
referencedEntities.addAll(references); convertDates(ruleIdentifier, name, entities, "dd/MM/yyyy");
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.category(category)
.value(value).transformation(transformation).references(new LinkedList<>(references)).build());
} }
public void create(String ruleIdentifier, String category, String value, String transformation, Entity reference) { public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities, String resultFormat) {
referencedEntities.add(reference); String valueDescription = "Convert values of type to dd/MM/yyyy joined with ', '";
List<Entity> referenceList = new LinkedList<>(); String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
referenceList.add(reference); create(ruleIdentifier, name, date, valueDescription, entities);
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.category(category)
.value(value)
.transformation(transformation)
.references(referenceList)
.build());
} }
public void create(String ruleIdentifier, String category, String value, String transformation) { public void create(String ruleIdentifier, String name, String value, String valueDescription) {
create(ruleIdentifier, category, value, transformation, Collections.emptyList()); create(ruleIdentifier, name, value, valueDescription, Collections.emptyList());
} }
public void create(String ruleIdentifier, String category, String value) { public void create(String ruleIdentifier, String name, String value) {
kieSession.insert(Component.builder() kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
.matchedRule(RuleIdentifier.fromString(ruleIdentifier)) .value(value).valueDescription("")
.category(category)
.value(value)
.transformation("")
.references(Collections.emptyList()) .references(Collections.emptyList())
.build()); .build());
} }

4
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl
View File

@ -75,7 +75,7 @@ rule "PerformingLaboratory.2.0: Performing Laboratory"
rule "PerformingLaboratory.0.2: Performing Laboratory" rule "PerformingLaboratory.0.2: Performing Laboratory"
salience -1 salience -1
when when
not Component(category == "Performing_Laboratory") not Component(name == "Performing_Laboratory")
then then
componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback"); componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback");
end end
@ -227,7 +227,7 @@ rule "DefaultComponents.999.0: Create components for all unmapped entities."
rule "X.0.0: merge duplicate component references" rule "X.0.0: merge duplicate component references"
when when
$first: Component() $first: Component()
$duplicate: Component(this != $first, category == $first.category, value == $first.value) $duplicate: Component(this != $first, name == $first.name, value == $first.value)
then then
$first.getReferences().addAll($duplicate.getReferences()); $first.getReferences().addAll($duplicate.getReferences());
retract($duplicate); retract($duplicate);