Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'DM-285' into 'master'

Browse Source 
DM-285: adjust ComponentLog to new Format

Closes DM-285

See merge request redactmanager/redaction-service!141
This commit is contained in:
Kilian Schüttler 2023-09-22 14:28:36 +02:00
commit ce580b6285
6 changed files with 100 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.70.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "8.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.181.0"
val persistenceServiceVersion = "2.182.0"
configurations {
all {

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java
View File

@ -20,9 +20,9 @@ import lombok.experimental.FieldDefaults;
public class Component {
RuleIdentifier matchedRule;
String category;
String name;
String value;
String transformation;
String valueDescription;
List<Entity> references;

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -197,7 +197,11 @@ public class AnalyzeService {
RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds, dictionary.getVersion());
document,
notFoundManualRedactionEntries,
previousEntityLog,
sectionsToReanalyseIds,
dictionary.getVersion());
return finalizeAnalysis(analyzeRequest,
startTime,
@ -294,10 +298,7 @@ public class AnalyzeService {
addedFileAttributes.stream().toList());
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(),
components,
dictionaryVersion,
kieWrapperComponentRules.rulesVersion());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);

37
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java
View File

@ -5,53 +5,50 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.EntityReference;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;
@Service
public class ComponentLogCreatorService {
public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, DictionaryVersion dictionaryVersion, long rulesVersion) {
public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
List<ComponentLogCategory> componentLogCategories = components.stream()
.collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.Component> componentLogComponents = components.stream()
.collect(Collectors.groupingBy(Component::getName, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
.entrySet()
.stream()
.map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue()))
.map(entry -> new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.Component(entry.getKey(), entry.getValue()))
.toList();
return new ComponentLog(analysisNumber, componentLogCategories, dictionaryVersion.getDossierVersion(), dictionaryVersion.getDossierTemplateVersion(), rulesVersion);
return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
}
private ComponentLogEntry buildComponentLogEntry(Component component) {
private ComponentValue buildComponentLogEntry(Component component) {
return ComponentLogEntry.builder()
.value(component.getValue()).matchedRule(component.getMatchedRule().toString())
.transformation(component.getTransformation())
.componentEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList()))
return ComponentValue.builder()
.value(component.getValue()).originalValue(component.getValue())
.componentRuleId(component.getMatchedRule().toString())
.valueDescription(component.getValueDescription())
.entityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList()))
.build();
}
private List<ComponentEntityReference> toComponentEntityReferences(List<Entity> references) {
private List<EntityReference> toComponentEntityReferences(List<Entity> references) {
return references.stream().map(this::toComponentEntityReference).toList();
}
private ComponentEntityReference toComponentEntityReference(Entity entity) {
private EntityReference toComponentEntityReference(Entity entity) {
return ComponentEntityReference.builder().id(entity.getId()).value(entity.getValue())
.page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0))
.reason(entity.getReason())
.ruleIdentifier(entity.getMatchedRule())
return EntityReference.builder().id(entity.getId())
.page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule())
.type(entity.getType())
.build();
}

150
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java
View File

@ -32,44 +32,50 @@ public class ComponentCreationService {
Set<Entity> referencedEntities = new HashSet<>();
public void firstOrElse(String ruleIdentifier, String category, Collection<Entity> entities, String fallback) {
public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
String transformation = String.format("First found value or else '%s'", fallback);
String valueDescription = String.format("First found value or else '%s'", fallback);
String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback);
create(ruleIdentifier, category, value, transformation, entities);
create(ruleIdentifier, name, value, valueDescription, entities);
}
public void joining(String ruleIdentifier, String category, Collection<Entity> entities) {
public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection<Entity> references) {
joining(ruleIdentifier, category, entities, ", ");
referencedEntities.addAll(references);
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
.value(value).valueDescription(valueDescription)
.references(new LinkedList<>(references))
.build());
}
public void joiningFromFirstSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities) {
public void joining(String ruleIdentifier, String name, Collection<Entity> entities) {
joiningFromFirstSectionOnly(ruleIdentifier, category, entities, ", ");
joining(ruleIdentifier, name, entities, ", ");
}
public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities) {
public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
joiningUniqueFromFirstSectionOnly(ruleIdentifier, category, entities, ", ");
}
public void joining(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
String transformation = String.format("Joining all values with '%s'", delimiter);
String valueDescription = String.format("Joining all values with '%s'", delimiter);
String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
create(ruleIdentifier, category, value, transformation, entities);
create(ruleIdentifier, name, value, valueDescription, entities);
}
public void joiningFromFirstSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
joiningFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
}
public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
joining(ruleIdentifier, category, entitiesFromFirstSection, delimiter);
joining(ruleIdentifier, name, entitiesFromFirstSection, delimiter);
}
@ -84,17 +90,16 @@ public class ComponentCreationService {
}
public void joiningFromLongestSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
joining(ruleIdentifier, category, entitiesFromLongestSection, delimiter);
joiningUniqueFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
}
public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
joiningUnique(ruleIdentifier, category, entitiesFromLongestSection, delimiter);
List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
joiningUnique(ruleIdentifier, name, entitiesFromFirstSection, delimiter);
}
@ -121,28 +126,35 @@ public class ComponentCreationService {
}
public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
joiningUnique(ruleIdentifier, category, entitiesFromFirstSection, delimiter);
}
public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities) {
joiningUnique(ruleIdentifier, category, entities, ", ");
}
public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
String transformation = String.format("Joining all values with '%s'", delimiter);
String valueDescription = String.format("Joining all values with '%s'", delimiter);
String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
create(ruleIdentifier, category, value, transformation, entities);
create(ruleIdentifier, name, value, valueDescription, entities);
}
public void asSentences(String ruleIdentifier, String category, Collection<Entity> entities) {
public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
joining(ruleIdentifier, name, entitiesFromLongestSection, delimiter);
}
public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
joiningUnique(ruleIdentifier, name, entitiesFromLongestSection, delimiter);
}
public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities) {
joiningUnique(ruleIdentifier, name, entities, ", ");
}
public void asSentences(String ruleIdentifier, String name, Collection<Entity> entities) {
if (entities.isEmpty()) {
return;
@ -153,23 +165,20 @@ public class ComponentCreationService {
iterator.setText(entity.getValue());
int start = iterator.first();
for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
create(ruleIdentifier, category, entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), "Split into sentences", entity);
create(ruleIdentifier, name, entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), "Split into sentences", entity);
}
}
}
public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities) {
public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) {
convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy");
}
public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities, String resultFormat) {
String transformation = "Convert values of type to dd/MM/yyyy joined with ', '";
String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
create(ruleIdentifier, category, date, transformation, entities);
referencedEntities.add(reference);
List<Entity> referenceList = new LinkedList<>();
referenceList.add(reference);
kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
.references(referenceList)
.build());
}
@ -181,45 +190,30 @@ public class ComponentCreationService {
}
public void create(String ruleIdentifier, String category, String value, String transformation, Collection<Entity> references) {
public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities) {
referencedEntities.addAll(references);
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.category(category)
.value(value).transformation(transformation).references(new LinkedList<>(references)).build());
convertDates(ruleIdentifier, name, entities, "dd/MM/yyyy");
}
public void create(String ruleIdentifier, String category, String value, String transformation, Entity reference) {
public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities, String resultFormat) {
referencedEntities.add(reference);
List<Entity> referenceList = new LinkedList<>();
referenceList.add(reference);
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.category(category)
.value(value)
.transformation(transformation)
.references(referenceList)
.build());
String valueDescription = "Convert values of type to dd/MM/yyyy joined with ', '";
String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
create(ruleIdentifier, name, date, valueDescription, entities);
}
public void create(String ruleIdentifier, String category, String value, String transformation) {
public void create(String ruleIdentifier, String name, String value, String valueDescription) {
create(ruleIdentifier, category, value, transformation, Collections.emptyList());
create(ruleIdentifier, name, value, valueDescription, Collections.emptyList());
}
public void create(String ruleIdentifier, String category, String value) {
public void create(String ruleIdentifier, String name, String value) {
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.category(category)
.value(value)
.transformation("")
kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
.value(value).valueDescription("")
.references(Collections.emptyList())
.build());
}

4
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl
View File

@ -75,7 +75,7 @@ rule "PerformingLaboratory.2.0: Performing Laboratory"
rule "PerformingLaboratory.0.2: Performing Laboratory"
salience -1
when
not Component(category == "Performing_Laboratory")
not Component(name == "Performing_Laboratory")
then
componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback");
end
@ -227,7 +227,7 @@ rule "DefaultComponents.999.0: Create components for all unmapped entities."
rule "X.0.0: merge duplicate component references"
when
$first: Component()
$duplicate: Component(this != $first, category == $first.category, value == $first.value)
$duplicate: Component(this != $first, name == $first.name, value == $first.value)
then
$first.getReferences().addAll($duplicate.getReferences());
retract($duplicate);