Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

DM-285: Entity to component mappings with new ruleset

Browse Source 
* refactoring structure
* added custom syntax validation, such that rule files may not be corrupted
* added component drools preparations
* deprecated redactionLog
* integrated entityLog
* integrated new getRules with ruleFileType
* renamed service.graph package -> service.document
* fixed tests
* update layout parser version
* initial component drools file - wip

Signed-off-by: Kilian Schuettler <kilian.schuettler@knecon.com>
This commit is contained in:
Kilian Schuettler 2023-09-08 17:11:39 +02:00
parent ee46e73c99
commit a8ce7ab1f3
32 changed files with 1346 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -12,11 +12,11 @@ plugins {
description = "redaction-service-server-v1"
val layoutParserVersion = "0.25.0"
val layoutParserVersion = "0.70.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "8.43.0.Final"
val pdfBoxVersion = "3.0.0-alpha2"
val persistenceServiceVersion = "DM-285-3"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "DM-285-5"
configurations {
all {

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java
View File

@ -5,7 +5,8 @@ import java.util.PriorityQueue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
@ -20,7 +21,7 @@ import lombok.experimental.FieldDefaults;
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualEntity implements Entity {
public class ManualEntity implements IEntity {
// The id must be mapped into a TextEntity as is for comments to work correctly
String id;
@ -88,4 +89,18 @@ public class ManualEntity implements Entity {
.build();
}
@Override
public TextRange getTextRange() {
return new TextRange(-1, -1);
}
@Override
public String type() {
return getManualOverwrite().getType().orElse(type);
}
}

41
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java Normal file
View File

@ -0,0 +1,41 @@
package com.iqser.red.service.redaction.v1.server.model.component;
import java.util.Collection;
import java.util.List;
import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class Component {
RuleIdentifier matchedRule;
String category;
String value;
String transformation;
List<Entity> references;
public boolean addReference(Entity entity) {
return references.add(entity);
}
public boolean addReferences(Collection<Entity> entities) {
return references.addAll(entities);
}
}

106
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java Normal file
View File

@ -0,0 +1,106 @@
package com.iqser.red.service.redaction.v1.server.model.component;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Getter
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class Entity {
String id;
String type;
EntryType entryType;
EntryState state;
String value;
String reason;
String matchedRule;
boolean rectangle;
String legalBasis;
boolean imported;
String section;
float[] color;
List<Position> positions;
int sectionNumber;
String textBefore;
String textAfter;
int startOffset;
int endOffset;
int length;
boolean imageHasTransparency;
boolean isDictionaryEntry;
boolean isDossierDictionaryEntry;
boolean excluded;
String sourceId;
List<Change> changes;
List<ManualChange> manualChanges;
Set<Engine> engines;
Set<String> reference;
Set<String> importedRedactionIntersections;
public static Entity fromEntityLogEntry(EntityLogEntry e) {
return Entity.builder()
.id(e.getId())
.type(e.getType())
.entryType(e.getEntryType())
.state(e.getState())
.value(e.getValue())
.reason(e.getReason())
.matchedRule(e.getMatchedRule())
.rectangle(e.isRectangle())
.legalBasis(e.getLegalBasis())
.imported(e.isImported())
.section(e.getSection())
.color(e.getColor())
.positions(e.getPositions())
.sectionNumber(e.getSectionNumber())
.textBefore(e.getTextBefore())
.textAfter(e.getTextAfter())
.startOffset(e.getStartOffset())
.endOffset(e.getEndOffset())
.length(e.getValue().length())
.imageHasTransparency(e.isImageHasTransparency())
.isDictionaryEntry(e.isDictionaryEntry())
.isDossierDictionaryEntry(e.isDossierDictionaryEntry())
.excluded(e.isExcluded())
.sourceId(e.getSourceId())
.changes(e.getChanges())
.manualChanges(e.getManualChanges())
.engines(e.getEngines())
.reference(e.getReference())
.importedRedactionIntersections(e.getImportedRedactionIntersections())
.build();
}
}

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java
View File

@ -1,24 +0,0 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
import java.util.List;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class Component {
String value;
String originalValue;
String transformation;
List<Entity> references;
}

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Entity.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java
View File

@ -5,11 +5,12 @@ import java.util.HashSet;
import java.util.PriorityQueue;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
import lombok.NonNull;
public interface Entity {
public interface IEntity {
PriorityQueue<MatchedRule> getMatchedRuleList();
@ -17,6 +18,27 @@ public interface Entity {
ManualChangeOverwrite getManualOverwrite();
String getValue();
TextRange getTextRange();
String type();
default int length() {
return value().length();
}
default String value() {
return getManualOverwrite().getValue().orElse(getValue());
}
// Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
default boolean applied() {

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
View File

@ -28,7 +28,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class TextEntity implements Entity {
public class TextEntity implements IEntity {
// primary key
@EqualsAndHashCode.Include
@ -201,5 +201,10 @@ public class TextEntity implements Entity {
}
@Override
public String type() {
return getManualOverwrite().getType().orElse(type);
}
}

39
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java
View File

@ -5,12 +5,14 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@ -30,7 +32,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class Image implements GenericSemanticNode, Entity {
public class Image implements GenericSemanticNode, IEntity {
List<Integer> treeId;
String id;
@ -77,6 +79,20 @@ public class Image implements GenericSemanticNode, Entity {
}
@Override
public TextRange getTextRange() {
return GenericSemanticNode.super.getTextRange();
}
@Override
public String type() {
return getManualOverwrite().getType().orElse(imageType.toString());
}
@Override
public String toString() {
@ -92,4 +108,23 @@ public class Image implements GenericSemanticNode, Entity {
return bBoxPerPage;
}
@Override
public String getValue() {
return NodeType.IMAGE + ":" + camelCase(imageType.toString());
}
private String camelCase(String name) {
return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH);
}
public int length() {
return 0;
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java
View File

@ -16,6 +16,6 @@ public enum NodeType {
public String toString() {
return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ENGLISH);
}
}

138
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
@ -35,15 +36,16 @@ import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
@ -71,6 +73,7 @@ public class AnalyzeService {
ComponentLogCreatorService componentLogCreatorService;
RedactionStorageService redactionStorageService;
RedactionChangeLogService redactionChangeLogService;
EntityChangeLogService entityChangeLogService;
LegalBasisClient legalBasisClient;
RedactionServiceSettings redactionServiceSettings;
ImportedRedactionService importedRedactionService;
@ -114,27 +117,14 @@ public class AnalyzeService {
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = new ComponentLog();
if (kieWrapperComponentRules.isPresent()) {
List<Component> components = componentDroolsExecutionService.executeRules(document.getEntities(),
allFileAttributes,
notFoundManualRedactionEntries,
document.streamAllImages().toList());
componentLog = componentLogCreatorService.buildComponentLog(kieWrapperComponentRules.container(), analyzeRequest.getAnalysisNumber(), components);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
return finalizeAnalysis(analyzeRequest,
startTime,
entityLog, componentLog,
startTime, kieWrapperComponentRules, entityLog,
redactionLog,
document.getNumberOfPages(),
dictionary.getVersion(),
false, new HashSet<>(allFileAttributes));
dictionary.getVersion(), false, new HashSet<>(allFileAttributes));
}
@ -184,16 +174,6 @@ public class AnalyzeService {
log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog previousComponentLog = new ComponentLog();
KieWrapper kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
if (kieWrapperComponentRules.isPresent()) {
log.info("Updated component rules to version {} for file {} in dossier {}",
kieWrapperComponentRules.rulesVersion(),
analyzeRequest.getFileId(),
analyzeRequest.getDossierId());
previousComponentLog = redactionStorageService.getComponentLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded previous component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -203,30 +183,28 @@ public class AnalyzeService {
}
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
new DictionaryVersion(previousRedactionLog.getDictionaryVersion(), previousRedactionLog.getDossierDictionaryVersion()),
new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()),
analyzeRequest.getDossierId());
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement);
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement);
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (sectionsToReAnalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest,
startTime,
previousEntityLog, previousComponentLog,
startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), previousEntityLog,
previousRedactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
true,
Collections.emptySet());
}
KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
@ -244,23 +222,11 @@ public class AnalyzeService {
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = new ComponentLog();
if (kieWrapperComponentRules.isPresent()) {
List<Component> components = componentDroolsExecutionService.executeRules(document.getEntities(),
allFileAttributes,
notFoundManualRedactionEntries,
document.streamAllImages().toList());
componentLog = componentLogCreatorService.buildComponentLog(kieWrapperComponentRules.container(), analyzeRequest.getAnalysisNumber(), components);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds);
return finalizeAnalysis(analyzeRequest,
startTime,
entityLog, componentLog,
startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLog,
redactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
@ -316,26 +282,20 @@ public class AnalyzeService {
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
EntityLog entityLog, ComponentLog componentLog,
long startTime, KieWrapper kieWrapperComponentRules, EntityLog entityLog,
RedactionLog redactionLog,
int numberOfPages,
DictionaryVersion dictionaryVersion,
boolean isReanalysis,
Set<FileAttribute> addedFileAttributes) {
redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
EntityLogChanges entityLogChanges = finalizeEntityLog(analyzeRequest, entityLog, redactionLog, dictionaryVersion);
RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
redactionLog,
analyzeRequest.getAnalysisNumber());
log.info("Created Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (entityLogChanges.isHasChanges()) {
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges);
}
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
log.info("Stored Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
long duration = System.currentTimeMillis() - startTime;
@ -346,13 +306,13 @@ public class AnalyzeService {
.fileId(analyzeRequest.getFileId())
.duration(duration)
.numberOfPages(numberOfPages)
.hasUpdates(redactionLogChange.isHasChanges())
.hasUpdates(entityLogChanges.isHasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(redactionLog.getRulesVersion())
.dictionaryVersion(redactionLog.getDictionaryVersion())
.legalBasisVersion(redactionLog.getLegalBasisVersion())
.dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
.rulesVersion(entityLog.getRulesVersion())
.dictionaryVersion(entityLog.getDictionaryVersion())
.legalBasisVersion(entityLog.getLegalBasisVersion())
.dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
.wasReanalyzed(isReanalysis)
.manualRedactions(analyzeRequest.getManualRedactions())
.addedFileAttributes(addedFileAttributes)
@ -360,16 +320,58 @@ public class AnalyzeService {
}
private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
KieWrapper kieWrapperComponentRules,
Set<FileAttribute> addedFileAttributes,
EntityLogChanges entityLogChanges) {
if (!kieWrapperComponentRules.isPresent()) {
return;
}
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
entityLogChanges.getEntityLog(),
addedFileAttributes.stream().toList());
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components);
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
}
private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, EntityLog entityLog, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) {
redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
entityLog,
analyzeRequest.getAnalysisNumber());
RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
redactionLog,
analyzeRequest.getAnalysisNumber());
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
return entityLogChanges;
}
private static List<SemanticNode> getSectionsToReAnalyse(Document document, Set<Integer> sectionsToReanalyseIds) {
return document.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTreeId().get(0))).collect(Collectors.toList());
}
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, Document document, DictionaryIncrement dictionaryIncrement) {
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) {
return analyzeRequest.getSectionsToReanalyse().isEmpty() //
? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, redactionLog, document, analyzeRequest) //
? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) //
: analyzeRequest.getSectionsToReanalyse();
}

71
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java
View File

@ -2,28 +2,30 @@ package com.iqser.red.service.redaction.v1.server.service;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;
@Service
public class ComponentLogCreatorService {
public ComponentLog buildComponentLog(KieContainer container, int analysisNumber, List<Component> components) {
public ComponentLog buildComponentLog(int analysisNumber, List<Component> components) {
List<ComponentLogEntry> componentLogEntries = components.stream().map(this::buildComponentLogEntry).toList();
return new ComponentLog(analysisNumber, componentLogEntries, -1, -1, -1);
List<ComponentLogCategory> componentLogCategories = components.stream()
.collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
.entrySet()
.stream()
.map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue()))
.toList();
return new ComponentLog(analysisNumber, componentLogCategories, -1, -1, -1);
}
@ -31,8 +33,7 @@ public class ComponentLogCreatorService {
return ComponentLogEntry.builder()
.value(component.getValue())
.transformation(component.getTransformation())
.originalValue(component.getOriginalValue())
.transformation(component.getTransformation()).originalValue(component.getReferences().stream().sorted(EntityComparators.start()).map(Entity::getValue).toList())
.componentEntityReferences(toComponentEntityReferences(component.getReferences()))
.build();
}
@ -40,45 +41,19 @@ public class ComponentLogCreatorService {
private List<ComponentEntityReference> toComponentEntityReferences(List<Entity> references) {
return references.stream().flatMap(this::toComponentEntityReference).toList();
return references.stream().map(this::toComponentEntityReference).toList();
}
private Stream<ComponentEntityReference> toComponentEntityReference(Entity entity) {
private ComponentEntityReference toComponentEntityReference(Entity entity) {
if (entity instanceof TextEntity textEntity) {
textEntity.getPositionsOnPagePerPage()
.stream()
.map(positionOnPage -> ComponentEntityReference.builder()
.page(positionOnPage.getPage().getNumber())
.type(textEntity.getType())
.id(positionOnPage.getId())
.ruleIdentifier(textEntity.getMatchedRule().getRuleIdentifier().toString())
.reason(textEntity.buildReasonWithManualChangeDescriptions())
.build());
} else if (entity instanceof Image image) {
return Stream.of(ComponentEntityReference.builder()
.id(image.getId())
.reason(image.buildReasonWithManualChangeDescriptions())
.type(image.getImageType().toString())
.ruleIdentifier(image.getMatchedRule().getRuleIdentifier().toString())
.page(image.getFirstPage().getNumber())
.build());
} else if (entity instanceof ManualEntity manualEntity) {
return manualEntity.getEntityPosition()
.stream()
.collect(Collectors.groupingBy(RectangleWithPage::pageNumber))
.keySet()
.stream()
.map(pageNumber -> ComponentEntityReference.builder()
.id(manualEntity.getId())
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.type(manualEntity.getType())
.ruleIdentifier(manualEntity.getRuleIdentifier())
.page(pageNumber)
.build());
}
throw new UnsupportedOperationException(String.format("Entity with class %s is not implemented yet", entity.getClass()));
return ComponentEntityReference.builder()
.id(entity.getId())
.page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0))
.reason(entity.getReason())
.ruleIdentifier(entity.getMatchedRule())
.type(entity.getType())
.build();
}
}

116
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java Normal file
View File

@ -0,0 +1,116 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class EntityChangeLogService {
private final RedactionStorageService redactionStorageService;
@Timed("redactmanager_computeChanges")
public EntityLogChanges computeChanges(String dossierId, String fileId, EntityLog currentEntityLog, int analysisNumber) {
long start = System.currentTimeMillis();
EntityLog previousEntityLog = redactionStorageService.getEntityLog(dossierId, fileId);
if (previousEntityLog == null) {
currentEntityLog.getEntityLogEntry().forEach(entry -> {
entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
});
return new EntityLogChanges(currentEntityLog, false);
}
List<EntityLogEntry> previouslyExistingEntries = previousEntityLog.getEntityLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList();
Map<String, EntityLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousEntityLog(currentEntityLog, previouslyExistingEntries);
Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(currentEntityLog, previouslyExistingEntries);
List<EntityLogEntry> newEntityLogEntries = previousEntityLog.getEntityLogEntry();
List<EntityLogEntry> toRemove = new ArrayList<>();
newEntityLogEntries.forEach(entry -> {
if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
List<Change> changes = entry.getChanges();
changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
var newEntry = addedEntryIds.get(entry.getId());
newEntry.setChanges(changes);
addedEntryIds.put(entry.getId(), newEntry);
toRemove.add(entry);
} else if (removedIds.contains(entry.getId())) {
entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
} else if (addedEntryIds.containsKey(entry.getId())) {
List<Change> changes = entry.getChanges();
changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
var newEntry = addedEntryIds.get(entry.getId());
newEntry.setChanges(changes);
addedEntryIds.put(entry.getId(), newEntry);
toRemove.add(entry);
}
});
newEntityLogEntries.removeAll(toRemove);
addedEntryIds.forEach((k, v) -> {
if (v.getChanges().isEmpty()) {
v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
}
newEntityLogEntries.add(v);
});
currentEntityLog.setEntityLogEntry(newEntityLogEntries);
log.debug("Change computation took: {}", System.currentTimeMillis() - start);
return new EntityLogChanges(currentEntityLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
}
private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
Set<EntityLogEntry> removed = new HashSet<>(previouslyExistingEntries);
currentEntityLog.getEntityLogEntry().forEach(removed::remove);
Set<String> removedIds = removed.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
return removedIds;
}
private static Map<String, EntityLogEntry> getEntriesThatExistInCurrentButNotInPreviousEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
Set<EntityLogEntry> currentExistingEntries = currentEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
.collect(Collectors.toSet());
previouslyExistingEntries.forEach(currentExistingEntries::remove);
Map<String, EntityLogEntry> addedIds = new HashMap<>();
currentExistingEntries.forEach(entry -> {
addedIds.put(entry.getId(), entry);
});
return addedIds;
}
}

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -16,7 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@ -188,13 +188,13 @@ public class EntityLogCreatorService {
}
private EntryState buildEntryState(Entity entity) {
private EntryState buildEntryState(IEntity IEntity) {
if (entity.applied()) {
if (IEntity.applied()) {
return EntryState.APPLIED;
} else if (entity.skipped()) {
} else if (IEntity.skipped()) {
return EntryState.SKIPPED;
} else if (entity.ignored()) {
} else if (IEntity.ignored()) {
return EntryState.IGNORED;
} else {
return EntryState.REMOVED;
@ -202,13 +202,13 @@ public class EntityLogCreatorService {
}
private EntryType buildEntryType(Entity entity, boolean isHint) {
private EntryType buildEntryType(IEntity IEntity, boolean isHint) {
if (entity instanceof TextEntity textEntity) {
if (IEntity instanceof TextEntity textEntity) {
return getEntryType(isHint, textEntity.getEntityType());
} else if (entity instanceof ManualEntity manualEntity) {
} else if (IEntity instanceof ManualEntity manualEntity) {
return getEntryType(isHint, manualEntity.getEntityType());
} else if (entity instanceof Image) {
} else if (IEntity instanceof Image) {
return EntryType.IMAGE;
}
throw new UnsupportedOperationException("Entity subclass %s not implemented!");

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java
View File

@ -7,7 +7,7 @@ import java.util.NoSuchElementException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
@ -24,14 +24,14 @@ public class ManualChangesApplicationService {
private final EntityCreationService entityCreationService;
public void recategorize(Entity entityToBeReCategorized, ManualRecategorization manualRecategorization) {
public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {
if (entityToBeReCategorized instanceof Image image) {
if (IEntityToBeReCategorized instanceof Image image) {
image.setImageType(ImageType.fromString(manualRecategorization.getType()));
return;
}
// need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed!
if (entityToBeReCategorized instanceof TextEntity textEntity) {
if (IEntityToBeReCategorized instanceof TextEntity textEntity) {
TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage());
recategorizedEntity.getManualOverwrite().addChange(manualRecategorization);

126
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java
View File

@ -1,5 +1,131 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.kie.api.runtime.KieSession;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
import com.iqser.red.service.redaction.v1.server.utils.DateConverter;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ComponentCreationService {
KieSession kieSession;
Set<Entity> referencedEntities = new HashSet<>();
public void firstOrElse(String ruleIdentifier, String category, Collection<Entity> entities, String fallback) {
String transformation = String.format("First found value or else '%s'", fallback);
String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback);
createComponent(ruleIdentifier, category, value, transformation, entities);
}
public void joining(String ruleIdentifier, String category, Collection<Entity> entities) {
joining(ruleIdentifier, category, entities, ", ");
}
public void joining(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
String transformation = String.format("Joining all values with '%s'", delimiter);
String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
createComponent(ruleIdentifier, category, value, transformation, entities);
}
public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities) {
joining(ruleIdentifier, category, entities, ", ");
}
public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
String transformation = String.format("Joining all values with '%s'", delimiter);
String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
createComponent(ruleIdentifier, category, value, transformation, entities);
}
public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities) {
convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy");
}
public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities, String resultFormat) {
String transformation = "Convert values of type to dd/MM/yyyy joined with ', '";
String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
createComponent(ruleIdentifier, category, date, transformation, entities);
}
public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
entities.forEach(entity -> createComponent(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
}
public void createComponent(String ruleIdentifier, String category, String value, String transformation, Collection<Entity> references) {
referencedEntities.addAll(references);
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.category(category)
.value(value)
.transformation(transformation)
.references(new LinkedList<>(references)));
}
public void createComponent(String ruleIdentifier, String category, String value, String transformation, Entity reference) {
referencedEntities.add(reference);
List<Entity> referenceList = new LinkedList<>();
referenceList.add(reference);
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.category(category)
.value(value)
.transformation(transformation)
.references(referenceList)
.build());
}
public void createComponent(String ruleIdentifier, String category, String value, String transformation) {
createComponent(ruleIdentifier, category, value, transformation, Collections.emptyList());
}
public void createComponent(String ruleIdentifier, String category, String value) {
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.category(category)
.value(value)
.transformation("")
.references(Collections.emptyList())
.build());
}
}

41
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java Normal file
View File

@ -0,0 +1,41 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.Comparator;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
public abstract class EntityComparators implements Comparator<Entity> {
private static class LongestEntity implements Comparator<Entity> {
@Override
public int compare(Entity Entity, Entity otherEntity) {
return Integer.compare(Entity.getLength(), otherEntity.getLength());
}
}
private static class FirstEntity implements Comparator<Entity> {
@Override
public int compare(Entity Entity, Entity otherEntity) {
return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset());
}
}
public static Comparator<Entity> length() {
return new LongestEntity();
}
public static Comparator<Entity> start() {
return new FirstEntity();
}
}

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/SectionFinderService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.service;
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.Collection;
import java.util.Collections;
@ -8,9 +8,11 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@ -19,13 +21,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel;
@ -34,18 +34,18 @@ import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
class SectionFinderService {
public class SectionFinderService {
@Timed("redactmanager_findSectionsToReanalyse")
public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Document document, AnalyzeRequest analyzeRequest) {
public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) {
long start = System.currentTimeMillis();
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
Set<Integer> sectionsToReanalyse = new HashSet<>();
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
sectionsToReanalyse.add(entry.getSectionNumber());
}

78
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java
View File

@ -1,16 +1,26 @@
package com.iqser.red.service.redaction.v1.server.service.drools;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.kie.api.runtime.KieContainer;
import org.kie.api.runtime.KieSession;
import org.kie.api.runtime.rule.QueryResults;
import org.kie.api.runtime.rule.QueryResultsRow;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -23,9 +33,61 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class ComponentDroolsExecutionService {
public List<Component> executeRules(Set<TextEntity> entities, List<FileAttribute> allFileAttributes, List<ManualEntity> notFoundManualRedactionEntries, List<Image> list) {
RedactionServiceSettings settings;
return Collections.emptyList();
public List<Component> executeRules(KieContainer kieContainer, EntityLog entityLog, List<FileAttribute> fileAttributes) {
KieSession kieSession = kieContainer.newKieSession();
ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
kieSession.setGlobal("componentCreationService", componentCreationService);
entityLog.getEntityLogEntry().stream().map(Entity::fromEntityLogEntry).forEach(kieSession::insert);
fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
kieSession.fireAllRules();
return null;
});
try {
completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
} catch (ExecutionException e) {
kieSession.dispose();
if (e.getCause() instanceof TimeoutException) {
throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
}
throw new RuntimeException(e);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
List<Component> components = getComponents(kieSession);
kieSession.dispose();
return components;
}
public List<FileAttribute> getFileAttributes(KieSession kieSession) {
List<FileAttribute> fileAttributes = new LinkedList<>();
QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes");
for (QueryResultsRow resultsRow : entitiesResult) {
fileAttributes.add((FileAttribute) resultsRow.get("$fileAttribute"));
}
return fileAttributes;
}
public List<Component> getComponents(KieSession kieSession) {
List<Component> components = new LinkedList<>();
QueryResults entitiesResult = kieSession.getQueryResults("getComponents");
for (QueryResultsRow resultsRow : entitiesResult) {
components.add((Component) resultsRow.get("$component"));
}
return components;
}
}

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -47,6 +47,16 @@ public class RedactionStorageService {
}
@SneakyThrows
public File getStoredObjectFile(String storageId) {
File tempFile = File.createTempFile("temp", "data");
tempFile.deleteOnExit();
storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
return tempFile;
}
@SneakyThrows
public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) {

49
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java Normal file
View File

@ -0,0 +1,49 @@
package com.iqser.red.service.redaction.v1.server.utils;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@UtilityClass
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DateConverter {
static List<SimpleDateFormat> formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH),
new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH),
new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH),
new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH),
new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH),
new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH));
public String convertDate(String dateAsString, String resultFormat) {
DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
Date date = null;
for (SimpleDateFormat format : formats) {
try {
date = format.parse(dateAsString);
break;
} catch (Exception e) {
log.warn("Failed to parse date from string {}. \n{}", dateAsString, e.getMessage());
// ignore, try next...
}
}
if (date == null) {
return dateAsString;
}
return resultDateFormat.format(date);
}
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -43,6 +43,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/documine_flora.drl");
private static final String COMPONENT_RULES = loadFromClassPath("drools/documine_flora_components.drl");
@Test
@ -148,7 +149,8 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES));
loadTypeForTest();
loadNerForTest();

9
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java
View File

@ -13,7 +13,6 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.io.MemoryUsageSetting;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
@ -36,10 +35,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException;
import lombok.RequiredArgsConstructor;
@ -67,14 +66,14 @@ public class AnnotationService {
*/
public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
var storedObjectFile = redactionStorageService.getStoredObjectFile(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
annotateRequest.getFileId(),
FileType.ORIGIN));
var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
try (PDDocument pdDocument = Loader.loadPDF(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
try (PDDocument pdDocument = Loader.loadPDF(storedObjectFile)) {
pdDocument.setAllSecurityToBeRemoved(true);
dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java
View File

@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest {
public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -255,7 +255,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
var tmpFileName = "/tmp/" + filename.split("/")[2] + "_ENTITY_BBOX.pdf";
var fileResource = new ClassPathResource(filename + ".pdf");
try (var fileStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(fileStream)) {
try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) {
for (Page page : document.getPages()) {
List<Rectangle2D> entityPositionsOnPage = page.getEntities()

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java
View File

@ -79,7 +79,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati
var tmpFileName = "/tmp/" + filename.split("/")[2] + "_SEMANTIC_NODES_BBOX.pdf";
var fileResource = new ClassPathResource(filename + ".pdf");
try (var fileStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(fileStream)) {
try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) {
PdfVisualisationUtility.drawDocumentGraph(pdDocument, document);
PdfVisualisationUtility.drawTextBlock(pdDocument,

9
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java
View File

@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.MockitoAnnotations;
import org.springframework.boot.test.mock.mockito.MockBean;
@ -154,4 +155,12 @@ class DroolsSyntaxValidationServiceTest {
assertFalse(droolsSyntaxValidation.isCompiled());
}
@Test
@SneakyThrows
@Disabled
void fixAllRuleFiles() {
}
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java
View File

@ -71,7 +71,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.textRange().start() < entity.textRange().end()));
ClassPathResource resource = new ClassPathResource(filePath);
try (PDDocument pdDocument = Loader.loadPDF(resource.getInputStream())) {
try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) {
Stream<NerEntities.NerEntity> unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document)
.getNerEntityList()
@ -120,7 +120,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getTextRange().start() < entity.getTextRange().end()));
ClassPathResource resource = new ClassPathResource(filePath);
try (PDDocument pdDocument = Loader.loadPDF(resource.getInputStream())) {
try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) {
List<TextEntity> validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document)
.getNerEntityList()

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java
View File

@ -21,6 +21,7 @@ public class LayoutParsingRequestProvider {
var positionBlockFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION);
var pageFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES);
var simplifiedTextStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SIMPLIFIED_TEXT);
var viewerDocumentStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.VIEWER_DOCUMENT);
return LayoutParsingRequest.builder()
.layoutParsingType(layoutParsingType)
.originFileStorageId(originFileStorageId)
@ -31,7 +32,7 @@ public class LayoutParsingRequestProvider {
.positionBlockFileStorageId(positionBlockFileStorageId)
.pageFileStorageId(pageFileStorageId)
.sectionGridStorageId(sectionGridStorageId)
.simplifiedTextStorageId(simplifiedTextStorageId)
.simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId)
.build();
}

558
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl Normal file
View File

@ -0,0 +1,558 @@
package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
global ComponentCreationService componentCreationService
//------------------------------------ queries ------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
query "getComponents"
$component: Component()
end
//------------------------------------ Default Components rules ------------------------------------
rule "StudyTitle.0.0: Study Title"
when
$titleCandidates: List() from collect (Entity(type == "study_title"))
then
componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!");
end
rule "PerformingLaboratory.0.0: Performing Laboratory"
when
$laboratoryName: Entity(type == "laboratory_name")
$laboratoryCountry: Entity(type == "laboratory_country", $distance: Math.abs($laboratoryName.startOffset - startOffset))
not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < $distance)
eval($distance < 80)
then
componentCreationService.createComponent("PerformingLaboratory.0.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!");
end
rule "PerformingLaboratory.0.1: Performing Laboratory"
when
$laboratoryName: Entity(type == "laboratory_name")
not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80)
then
componentCreationService.createComponent("DefaultComponents.0.1", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!");
end
rule "PerformingLaboratory.0.2: Performing Laboratory"
salience -1
when
not Component(category == "Performing_Laboratory")
then
componentCreationService.createComponent("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback");
end
rule "ReportNumber.0.0: Report number"
when
$reportNumberCandidates: List() from collect (Entity(type == "report_number"))
then
componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "No report number found!");
end
rule "GLPStudy.0.0: GLP Study"
when
$glpStudy: Entity(type() == "glp_study")
then
componentCreationService.createComponent("GLPStudy.0.0", "GLP_study", "Yes", "Yes if present, No if not", $glpStudy);
end
rule "GLPStudy.1.0: GLP Study"
when
not Entity(type() == "glp_study")
then
componentCreationService.createComponent("GLPStudy.1.0", "GLP_study", "No", "Yes if present, No if not");
end
rule "GLPStudy.2.0: GLP Study"
when
$first: Component(category == "GLP_study")
$second: Component(category == "GLP_study", value == $first.value, this != $first)
then
$first.addReferences($second.getReferences());
retract($second);
end
rule "DefaultComponents.4.0: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number")
not Entity(type == "oecd_guideline_year")
then
componentCreationService.createComponent("DefaultComponents.4.0", "Test_Guidelines_1", $guidelineNumber.getValue(), "Only OECD Number found!");
end
rule "DefaultComponents.4.1: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2015")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "404")
then
componentCreationService.createComponent(
"DefaultComponents.4.1",
"Test_Guidelines_1",
"Nº 404: Acute Dermal Irritation/Corrosion (28/07/2015)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.2: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "1981")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "403")
then
componentCreationService.createComponent(
"DefaultComponents.4.2",
"Test_Guidelines_1",
"Nº 403: Acute Inhalation Toxicity (12/05/1981)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.3: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "442B")
then
componentCreationService.createComponent(
"DefaultComponents.4.3",
"Test_Guidelines_1",
"Nº 442B: Skin Sensitization (27/06/2018)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.4: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2016")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "487")
then
componentCreationService.createComponent(
"DefaultComponents.4.4",
"Test_Guidelines_1",
"Nº 487: Micronucleus Human Lymphocytes (2016)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.5: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2004")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "428")
then
componentCreationService.createComponent(
"DefaultComponents.4.5",
"Test_Guidelines_1",
"Nº 428: Split-Thickness Skin test (2004)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.6: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "442A")
then
componentCreationService.createComponent(
"DefaultComponents.4.6",
"Test_Guidelines_1",
"Nº 442A: Skin Sensitization (23/07/2018)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.7: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "404")
then
componentCreationService.createComponent(
"DefaultComponents.4.7",
"Test_Guidelines_1",
"Nº 404: Acute Dermal Irritation/Corrosion (24/04/2002)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.8: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "1992")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "406")
then
componentCreationService.createComponent(
"DefaultComponents.4.8",
"Test_Guidelines_1",
"Nº 406: Skin Sensitisation (1992)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.9: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "438")
then
componentCreationService.createComponent(
"DefaultComponents.4.9",
"Test_Guidelines_1",
"Nº 438: Eye Irritation (26/06/2018)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.10: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2009")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "436")
then
componentCreationService.createComponent(
"DefaultComponents.4.10",
"Test_Guidelines_1",
"Nº 436: Acute Inhalation Toxicity – Acute Toxic Class Method (08/09/2009)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.11: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "429")
then
componentCreationService.createComponent(
"DefaultComponents.4.11",
"Test_Guidelines_1",
"Nº 429: Skin Sensitisation: Local Lymph Node Assay (24/04/2002)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.12: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2020")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "471")
then
componentCreationService.createComponent(
"DefaultComponents.4.12",
"Test_Guidelines_1",
"Nº 471: Bacterial Reverse Mutation Test (26/06/2020)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.13: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "1987")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "405")
then
componentCreationService.createComponent(
"DefaultComponents.4.13",
"Test_Guidelines_1",
"Nº 405: Acute Eye Irritation/Corrosion (24/02/1987)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.14: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "1997")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "471")
then
componentCreationService.createComponent(
"DefaultComponents.4.14",
"Test_Guidelines_1",
"Nº 471: Bacterial Reverse Mutation Test (21/07/1997)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.15: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2009")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "403")
then
componentCreationService.createComponent(
"DefaultComponents.4.15",
"Test_Guidelines_1",
"Nº 403: Acute Inhalation Toxicity (08/09/2009)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.16: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "433")
then
componentCreationService.createComponent(
"DefaultComponents.4.16",
"Test_Guidelines_1",
"Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (27/06/2018)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.17: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2019")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "439")
then
componentCreationService.createComponent(
"DefaultComponents.4.17",
"Test_Guidelines_1",
"Nº 439: Skin Irritation (2019)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.18: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2001")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "425")
then
componentCreationService.createComponent(
"DefaultComponents.4.18",
"Test_Guidelines_1",
"Nº 425: Acute oral Toxicity - Up-and-Down Procedure (17/12/2001)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.19: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "402")
then
componentCreationService.createComponent(
"DefaultComponents.4.19",
"Test_Guidelines_1",
"Nº 402: Acute Dermal Toxicity (09/10/2017)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.20: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "1981")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "404")
then
componentCreationService.createComponent(
"DefaultComponents.4.20",
"Test_Guidelines_1",
"Nº 404: Acute Dermal Irritation/Corrosion (12/05/1981)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.21: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2016")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "474")
then
componentCreationService.createComponent(
"DefaultComponents.4.21",
"Test_Guidelines_1",
"Nº 474: Micronucleus Bone Marrow Cells Rat (2016)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.22: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "433")
then
componentCreationService.createComponent(
"DefaultComponents.4.22",
"Test_Guidelines_1",
"Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (09/10/2017)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.23: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "1992")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "404")
then
componentCreationService.createComponent(
"DefaultComponents.4.23",
"Test_Guidelines_1",
"Nº 404: Acute Dermal Irritation/Corrosion (17/07/1992)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.24: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2010")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "429")
then
componentCreationService.createComponent(
"DefaultComponents.4.24",
"Test_Guidelines_1",
"Nº 429: Skin Sensitisation (23/07/2010)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.25: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "405")
then
componentCreationService.createComponent(
"DefaultComponents.4.25",
"Test_Guidelines_1",
"Nº 405: Acute Eye Irritation/Corrosion (09/10/2017)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.26: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "1987")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "402")
then
componentCreationService.createComponent(
"DefaultComponents.4.26",
"Test_Guidelines_1",
"Nº 402: Acute Dermal Toxicity (24/02/1987)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.27: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2012")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "405")
then
componentCreationService.createComponent(
"DefaultComponents.4.27",
"Test_Guidelines_1",
"Nº 405: Acute Eye Irritation/Corrosion (02/10/2012)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.28: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "405")
then
componentCreationService.createComponent(
"DefaultComponents.4.28",
"Test_Guidelines_1",
"Nº 405: Acute Eye Irritation/Corrosion (24/04/2002)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.4.29: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", value == "2008")
$guidelineYear: Entity(type == "oecd_guideline_year", value == "425")
then
componentCreationService.createComponent(
"DefaultComponents.4.29",
"Test_Guidelines_1",
"Nº 425: Acute oral Toxicity - Up-and-Down Procedure (03/10/2008)",
"OECD Number and guideline year mapped!"
);
end
rule "DefaultComponents.5.0: Test Guideline 2"
when
$epaGuideLines: List() from collect (Entity(type() == "epa_guideline"))
$ecGuideLines: List() from collect (Entity(type() == "ec_guideline"))
then
componentCreationService.joining("DefaultComponents.5.0",
"Test_Guideline_2",
Stream.of(
$epaGuideLines.stream(),
$ecGuideLines.stream())
.flatMap(a -> a)
.toList()
);
end
rule "DefaultComponents.6.0: Experimental Starting Date"
when
$startDates: List(!isEmpty()) from collect (Entity(type() == "experimental_start_date"))
then
componentCreationService.convertDates("DefaultComponents.6.0", "Experimental_Starting_Date", $startDates);
end
rule "DefaultComponents.7.0: Experimental Completion Date"
when
$endDates: List(!isEmpty()) from collect (Entity(type() == "experimental_end_date"))
then
componentCreationService.convertDates("DefaultComponents.7.0", "Experimental_Completion_Date", $endDates);
end
rule "DefaultComponents.8.0: Certificate of analysis batch identification"
when
$batchNumbers: List(!isEmpty()) from collect (Entity(type() == "batch_number"))
then
componentCreationService.joiningUnique("DefaultComponents.8.0", "Batch_Number", $batchNumbers);
end
rule "DefaultComponents.999.0: Create components for all unmapped entities."
salience -999
when
$allEntities: List(!isEmpty()) from collect (Entity()
then
componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
end
//------------------------------------ Component merging rules ------------------------------------
rule "X.0.0: merge duplicate component references"
when
$first: Component()
$duplicate: Component(this != $first, category == category, value == value)
then
$first.getReferences().addAll($duplicate.getReferences());
retract($duplicate);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -58,6 +58,7 @@ global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
//------------------------------------ queries ------------------------------------
query "getFileAttributes"

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -58,6 +58,7 @@ global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------
query "getFileAttributes"

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -47,7 +47,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType
import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType
global Document document
global EntityCreationService entityCreationService