Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7631' into 'master'

Browse Source 
RED-7631: create EntityLog to replace RedactionLog

Closes RED-7631

See merge request redactmanager/redaction-service!152
This commit is contained in:
Ali Oezyetimoglu 2023-10-09 17:16:59 +02:00
commit d71f06d884
21 changed files with 351 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
publish-custom-image.sh
View File

@ -9,7 +9,7 @@ branch=$(git rev-parse --abbrev-ref HEAD)
commit_hash=$(git rev-parse --short=5 HEAD) commit_hash=$(git rev-parse --short=5 HEAD)
# Combine branch and commit hash # Combine branch and commit hash
buildName="${branch}-${commit_hash}" buildName="${USER}-${branch}-${commit_hash}"
gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName" echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java
View File

@ -56,8 +56,7 @@ public class ManualEntity implements IEntity {
.reason(redactionLogEntry.getReason()) .reason(redactionLogEntry.getReason())
.legalBasis(redactionLogEntry.getLegalBasis()) .legalBasis(redactionLogEntry.getLegalBasis())
.type(redactionLogEntry.getType()) .type(redactionLogEntry.getType())
.section(redactionLogEntry.getSection()) .section(redactionLogEntry.getSection()).entityType(getEntityType(redactionLogEntry))
.entityType(redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY)
.applied(redactionLogEntry.isRedacted()) .applied(redactionLogEntry.isRedacted())
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry()) .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
.isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry()) .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
@ -66,7 +65,19 @@ public class ManualEntity implements IEntity {
} }
public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry) { private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.isRecommendation()) {
return EntityType.RECOMMENDATION;
}
if (redactionLogEntry.isHint()) {
return EntityType.HINT;
}
return EntityType.ENTITY;
}
public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList(); List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(); ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite();
@ -79,8 +90,7 @@ public class ManualEntity implements IEntity {
.reason(manualRedactionEntry.getReason()) .reason(manualRedactionEntry.getReason())
.legalBasis(manualRedactionEntry.getLegalBasis()) .legalBasis(manualRedactionEntry.getLegalBasis())
.type(manualRedactionEntry.getType()) .type(manualRedactionEntry.getType())
.section(manualRedactionEntry.getSection()) .section(manualRedactionEntry.getSection()).entityType(hint ? EntityType.HINT : EntityType.ENTITY)
.entityType(EntityType.ENTITY)
.applied(true) .applied(true)
.isDictionaryEntry(false) .isDictionaryEntry(false)
.isDossierDictionaryEntry(false) .isDossierDictionaryEntry(false)

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity;
public enum EntityType { public enum EntityType {
ENTITY, ENTITY,
HINT,
RECOMMENDATION, RECOMMENDATION,
FALSE_POSITIVE, FALSE_POSITIVE,
FALSE_RECOMMENDATION FALSE_RECOMMENDATION

118
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -81,56 +81,6 @@ public class AnalyzeService {
FunctionTimerValues redactmanagerAnalyzePagewiseValues; FunctionTimerValues redactmanagerAnalyzePagewiseValues;
@Timed("redactmanager_analyze")
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
dictionarySearchService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
dictionary,
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
EntityLog entityLog = entityLogCreatorService.createInitialEntityLogFromActiveEntities(analyzeRequest,
document,
notFoundManualRedactionEntries,
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
redactionLog,
document.getNumberOfPages(),
dictionary.getVersion(),
false,
new HashSet<>(allFileAttributes));
}
@Timed("redactmanager_reanalyze") @Timed("redactmanager_reanalyze")
@SneakyThrows @SneakyThrows
public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
@ -164,8 +114,7 @@ public class AnalyzeService {
kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog, entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
dictionaryIncrement.getDictionaryVersion(), dictionaryIncrement.getDictionaryVersion(),
analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierTemplateId(), false), document,
false), document,
previousRedactionLog, previousRedactionLog,
document.getNumberOfPages(), document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(), dictionaryIncrement.getDictionaryVersion(),
@ -178,7 +127,9 @@ public class AnalyzeService {
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document); List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document,
analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -205,8 +156,7 @@ public class AnalyzeService {
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLogChanges, document,
entityLogChanges, document,
redactionLog, redactionLog,
document.getNumberOfPages(), document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(), dictionaryIncrement.getDictionaryVersion(),
@ -215,6 +165,58 @@ public class AnalyzeService {
} }
@Timed("redactmanager_analyze")
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document,
analyzeRequest.getDossierTemplateId());
dictionarySearchService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
dictionary,
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
document,
notFoundManualRedactionEntries,
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
redactionLog,
document.getNumberOfPages(),
dictionary.getVersion(),
false,
new HashSet<>(allFileAttributes));
}
@Deprecated(forRemoval = true) @Deprecated(forRemoval = true)
private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest, private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
@ -283,8 +285,7 @@ public class AnalyzeService {
} }
private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document,
KieWrapper kieWrapperComponentRules, Document document,
Set<FileAttribute> addedFileAttributes, Set<FileAttribute> addedFileAttributes,
EntityLogChanges entityLogChanges, EntityLogChanges entityLogChanges,
DictionaryVersion dictionaryVersion) { DictionaryVersion dictionaryVersion) {
@ -293,8 +294,7 @@ public class AnalyzeService {
return; return;
} }
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document,
entityLogChanges.getEntityLog(), document,
addedFileAttributes.stream().toList()); addedFileAttributes.stream().toList());
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -27,7 +27,7 @@ public class DictionarySearchService {
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
for (var model : dictionary.getDictionaryModels()) { for (var model : dictionary.getDictionaryModels()) {
bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
} }

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -29,13 +30,14 @@ public class EntityChangeLogService {
var now = OffsetDateTime.now(); var now = OffsetDateTime.now();
for (EntityLogEntry entityLogEntry : newEntityLogEntries) { for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
var optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny(); Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
if (optionalPreviousEntity.isEmpty()) { if (optionalPreviousEntity.isEmpty()) {
hasChanges = true; hasChanges = true;
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)); entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
continue; continue;
} }
var previousEntity = optionalPreviousEntity.get(); var previousEntity = optionalPreviousEntity.get();
entityLogEntry.getChanges().addAll(previousEntity.getChanges());
if (!previousEntity.getState().equals(entityLogEntry.getState())) { if (!previousEntity.getState().equals(entityLogEntry.getState())) {
hasChanges = true; hasChanges = true;
ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState()); ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
@ -54,25 +56,22 @@ public class EntityChangeLogService {
private ChangeType calculateChangeType(EntryState state, EntryState previousState) { private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
if (state.equals(previousState)) {
throw new IllegalArgumentException("States are equal, can't calculate ChangeType.");
}
if (!isRemoved(previousState) && isRemoved(state)) { if (!isRemoved(previousState) && isRemoved(state)) {
return ChangeType.REMOVED; return ChangeType.REMOVED;
} }
if (isRemoved(previousState) && !isRemoved(state)) { if (isRemoved(previousState) && !isRemoved(state)) {
return ChangeType.ADDED; return ChangeType.ADDED;
} }
if (previousState.equals(EntryState.APPLIED) && state.equals(EntryState.SKIPPED)) {
return ChangeType.CHANGED; return ChangeType.CHANGED;
} }
if (previousState.equals(EntryState.SKIPPED) && state.equals(EntryState.APPLIED)) {
return ChangeType.CHANGED;
}
return null;
}
private static boolean isRemoved(EntryState state) { private static boolean isRemoved(EntryState state) {
return state.equals(EntryState.IGNORED) || state.equals(EntryState.REMOVED); return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED));
} }

149
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -50,13 +50,19 @@ public class EntityLogCreatorService {
private final EntityChangeLogService entityChangeLogService; private final EntityChangeLogService entityChangeLogService;
public EntityLog createInitialEntityLogFromActiveEntities(AnalyzeRequest analyzeRequest, private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION));
}
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
List<ManualEntity> notFoundManualRedactionEntries, List<ManualEntity> notFoundManualRedactionEntries,
DictionaryVersion dictionaryVersion, DictionaryVersion dictionaryVersion,
long rulesVersion) { long rulesVersion) {
List<EntityLogEntry> entityLogEntries = createEntityLogEntriesFromActiveEntities(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
@ -83,36 +89,6 @@ public class EntityLogCreatorService {
} }
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
EntityLog previousEntityLog,
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> entriesFromReanalysedSections = previousEntityLog.getEntityLogEntry()
.stream().filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
.equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
.toList();
previousEntityLog.getEntityLogEntry().removeAll(entriesFromReanalysedSections);
boolean hasChanges = entityChangeLogService.computeChanges(entriesFromReanalysedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newEntityLogEntries,
false);
previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages());
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges);
}
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) { public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) {
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId); List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId);
@ -125,34 +101,47 @@ public class EntityLogCreatorService {
} }
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
EntityLog previousEntityLog,
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
.equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
.toList();
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
var newEntityLogWithImportedEntities = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newEntityLogEntries,
false);
previousEntityLog.getEntityLogEntry().addAll(newEntityLogWithImportedEntities);
excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages());
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges);
}
private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) { private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
List<EntityLogEntry> entries = new ArrayList<>(); List<EntityLogEntry> entries = new ArrayList<>();
document.getEntities() document.getEntities()
.stream() .stream().filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation).filter(entity -> !entity.removed())
.filter(EntityLogCreatorService::isEntityOrRecommendationType)
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
document.streamAllImages().forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId)));
return entries; return entries;
}
private List<EntityLogEntry> createEntityLogEntriesFromActiveEntities(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
List<EntityLogEntry> entries = new ArrayList<>();
document.getEntities().stream().filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active)
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
document.streamAllImages().filter(IEntity::active).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.stream().filter(IEntity::active).forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
return entries;
}
private static boolean isEntityOrRecommendationType(TextEntity textEntity) {
return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION;
} }
@ -189,8 +178,7 @@ public class EntityLogCreatorService {
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return EntityLogEntry.builder() return EntityLogEntry.builder()
.id(image.getId()) .id(image.getId())
.value(image.value()) .value(image.value()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
.color(getColor(imageType, dossierTemplateId, image.applied()))
.value(image.value()) .value(image.value())
.type(imageType) .type(imageType)
.reason(image.buildReasonWithManualChangeDescriptions()) .reason(image.buildReasonWithManualChangeDescriptions())
@ -203,8 +191,7 @@ public class EntityLogCreatorService {
.section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
.imageHasTransparency(image.isTransparent()) .imageHasTransparency(image.isTransparent())
.manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(image)) .state(buildEntryState(image)).entryType(buildEntryType(image))
.entryType(buildEntryType(image, isHint))
.build(); .build();
} }
@ -213,16 +200,14 @@ public class EntityLogCreatorService {
private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) { private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(type, dossierTemplateId); boolean isHint = isHint(manualEntity.getEntityType());
return EntityLogEntry.builder() return EntityLogEntry.builder()
.id(manualEntity.getId()) .id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
.color(getColor(type, dossierTemplateId, manualEntity.applied()))
.reason(manualEntity.buildReasonWithManualChangeDescriptions()) .reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis()) .legalBasis(manualEntity.legalBasis())
.value(manualEntity.value()) .value(manualEntity.value())
.type(type) .type(type)
.state(buildEntryState(manualEntity)) .state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity))
.entryType(buildEntryType(manualEntity, isHint))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.containingNodeId(Collections.emptyList()) .containingNodeId(Collections.emptyList())
.closestHeadline("") .closestHeadline("")
@ -245,10 +230,8 @@ public class EntityLogCreatorService {
Set<String> referenceIds = new HashSet<>(); Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); boolean isHint = isHint(entity.getEntityType());
boolean isHint = isHint(entity.getType(), dossierTemplateId); return EntityLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
return EntityLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
.reason(entity.buildReasonWithManualChangeDescriptions()) .reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis()) .legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@ -266,27 +249,26 @@ public class EntityLogCreatorService {
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
.reference(referenceIds) .reference(referenceIds)
.manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(entity)) .state(buildEntryState(entity)).entryType(buildEntryType(entity))
.entryType(buildEntryType(entity, isHint))
.build(); .build();
} }
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { private boolean isHint(EntityType entityType) {
if (!isRedaction && !isHint(type, dossierTemplateId)) { return entityType.equals(EntityType.HINT);
}
private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
if (!isApplied && !isHint) {
return dictionaryService.getNotRedactedColor(dossierTemplateId); return dictionaryService.getNotRedactedColor(dossierTemplateId);
} }
return dictionaryService.getColor(type, dossierTemplateId); return dictionaryService.getColor(type, dossierTemplateId);
} }
private boolean isHint(String type, String dossierTemplateId) {
return dictionaryService.isHint(type, dossierTemplateId);
}
private EntryState buildEntryState(IEntity entity) { private EntryState buildEntryState(IEntity entity) {
if (entity.applied() && entity.active()) { if (entity.applied() && entity.active()) {
@ -301,26 +283,27 @@ public class EntityLogCreatorService {
} }
private EntryType buildEntryType(IEntity entity, boolean isHint) { private EntryType buildEntryType(IEntity entity) {
if (entity instanceof TextEntity textEntity) { if (entity instanceof TextEntity textEntity) {
return getEntryType(isHint, textEntity.getEntityType()); return getEntryType(textEntity.getEntityType());
} else if (entity instanceof ManualEntity manualEntity) { } else if (entity instanceof ManualEntity manualEntity) {
if (((ManualEntity) entity).isRectangle()) { if (((ManualEntity) entity).isRectangle()) {
return EntryType.AREA; return EntryType.AREA;
} }
return getEntryType(isHint, manualEntity.getEntityType()); return getEntryType(manualEntity.getEntityType());
} else if (entity instanceof Image) { } else if (entity instanceof Image) {
return EntryType.IMAGE; return EntryType.IMAGE;
} }
throw new UnsupportedOperationException("Entity subclass %s not implemented!"); throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
} }
private static EntryType getEntryType(boolean isHint, EntityType entityType) { private static EntryType getEntryType(EntityType entityType) {
return switch (entityType) { return switch (entityType) {
case ENTITY -> isHint ? EntryType.HINT : EntryType.ENTITY; case ENTITY -> EntryType.ENTITY;
case HINT -> EntryType.HINT;
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
case RECOMMENDATION -> EntryType.RECOMMENDATION; case RECOMMENDATION -> EntryType.RECOMMENDATION;
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java
View File

@ -16,13 +16,15 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor @RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualChangesApplicationService { public class ManualChangesApplicationService {
private final EntityCreationService entityCreationService; EntityCreationService entityCreationService;
public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) { public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {

128
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java
View File

@ -14,14 +14,15 @@ import java.util.stream.Collectors;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
@ -30,7 +31,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -51,8 +51,7 @@ public class RedactionLogCreatorService {
List<RedactionLogEntry> entries = new ArrayList<>(); List<RedactionLogEntry> entries = new ArrayList<>();
document.getEntities() document.getEntities()
.stream() .stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(RedactionLogCreatorService::isEntityOrRecommendationType)
.filter(IEntity::active) .filter(IEntity::active)
.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments))); .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments)));
document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments))); document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
@ -61,9 +60,9 @@ public class RedactionLogCreatorService {
} }
private static boolean isEntityOrRecommendationType(TextEntity textEntity) { private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION; return !(textEntity.getEntityType() == EntityType.FALSE_POSITIVE || textEntity.getEntityType() == EntityType.FALSE_RECOMMENDATION);
} }
@ -133,9 +132,8 @@ public class RedactionLogCreatorService {
Set<String> referenceIds = new HashSet<>(); Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
boolean isHint = isHint(entity.getType(), dossierTemplateId); boolean isHint = isHint(entity.getEntityType());
return RedactionLogEntry.builder() return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
.color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
.reason(entity.buildReasonWithManualChangeDescriptions()) .reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis()) .legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@ -177,39 +175,9 @@ public class RedactionLogCreatorService {
} }
public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) { private boolean isHint(EntityType entityType) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); return entityType.equals(EntityType.HINT);
boolean isHint = isHint(type, dossierTemplateId);
return RedactionLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied()))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
.type(type)
.redacted(manualEntity.applied())
.isHint(isHint)
.isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.sectionNumber(0).matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
.rectangle(manualEntity.isRectangle())
.isDictionaryEntry(manualEntity.isDictionaryEntry())
.isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.textAfter("")
.textBefore("")
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getEntityPosition()
.stream()
.map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.collect(Collectors.toList()))
.engines(Collections.emptySet())
.reference(Collections.emptySet())
.manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
.comments(buildRedactionLogComments(comments, manualEntity.getId()))
.build();
} }
@ -242,13 +210,64 @@ public class RedactionLogCreatorService {
} }
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean isHint) {
if (!isRedaction && isHint) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(manualEntity.getEntityType());
return RedactionLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
.type(type)
.redacted(manualEntity.applied())
.isHint(isHint)
.isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.sectionNumber(0)
.matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
.rectangle(manualEntity.isRectangle())
.isDictionaryEntry(manualEntity.isDictionaryEntry())
.isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.textAfter("")
.textBefore("")
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getEntityPosition()
.stream()
.map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.collect(Collectors.toList()))
.engines(Collections.emptySet())
.reference(Collections.emptySet())
.manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
.comments(buildRedactionLogComments(comments, manualEntity.getId()))
.build();
}
private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
}
public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) { public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.id(image.getId()) .id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
.color(getColor(imageType, dossierTemplateId, image.applied()))
.isImage(true) .isImage(true)
.value(image.value()) .value(image.value())
.type(imageType) .type(imageType)
@ -269,27 +288,4 @@ public class RedactionLogCreatorService {
} }
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
if (!isRedaction && !isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private boolean isHint(String type, String dossierTemplateId) {
return dictionaryService.isHint(type, dossierTemplateId);
}
private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()),
(float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(),
pageNumber);
}
} }

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java
View File

@ -29,21 +29,27 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionO
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Service @Service
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualEntityCreationService { public class ManualEntityCreationService {
private static final double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities static double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities
private final EntityCreationService entityCreationService; EntityCreationService entityCreationService;
DictionaryService dictionaryService;
@Autowired @Autowired
public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService) { public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
entityCreationService = new EntityCreationService(entityEnrichmentService); entityCreationService = new EntityCreationService(entityEnrichmentService);
this.dictionaryService = dictionaryService;
} }
@ -60,11 +66,14 @@ public class ManualEntityCreationService {
} }
public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries, SemanticNode node) { public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries,
SemanticNode node,
String dossierTemplateId) {
List<ManualEntity> manualEntities = manualRedactionEntries.stream() List<ManualEntity> manualEntities = manualRedactionEntries.stream()
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary())) .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
.map(ManualEntity::fromManualRedactionEntry) .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.peek(manualEntity -> manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis())) .peek(manualEntity -> manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis()))
.toList(); .toList();

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java
View File

@ -24,12 +24,12 @@ public class ManualRedactionEntryService {
private final ManualEntityCreationService manualEntityCreationService; private final ManualEntityCreationService manualEntityCreationService;
public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document) { public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList(); List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList();
if (analyzeRequest.getManualRedactions() != null) { if (analyzeRequest.getManualRedactions() != null) {
notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions() notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions()
.getEntriesToAdd(), document); .getEntriesToAdd(), document, dossierTemplateId);
log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
} }
if (notFoundManualRedactionEntries.isEmpty()) { if (notFoundManualRedactionEntries.isEmpty()) {

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -1,7 +1,6 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.util.List; import java.util.List;
@ -26,7 +25,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java
View File

@ -9,6 +9,7 @@ import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.mockito.MockitoAnnotations; import org.mockito.MockitoAnnotations;
import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.MockBean;
@ -210,6 +211,7 @@ class DroolsSyntaxValidationServiceTest {
@Test @Test
@SneakyThrows @SneakyThrows
@Disabled
void attemptImportsFixToAllRuleFiles() { void attemptImportsFixToAllRuleFiles() {
DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));

36
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -15,9 +15,6 @@ import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -77,7 +74,7 @@ import lombok.SneakyThrows;
@Import(ManualChangesEnd2EndTest.TestConfiguration.class) @Import(ManualChangesEnd2EndTest.TestConfiguration.class)
public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules.drl"); private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl"); private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl");
@Autowired @Autowired
private EntityEnrichmentService entityEnrichmentService; private EntityEnrichmentService entityEnrichmentService;
@ -297,20 +294,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst() .findFirst()
.get(); .get();
var mouses = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals("vertebrate")).filter(entry -> entry.getValue().equals("Mouse")).toList(); assertEquals("CBI.7.0", asyaLyon.getMatchedRule());
assertEquals("Published Information found in section", asyaLyon.getReason());
var recategorizations = mouses.stream() assertFalse(asyaLyon.isRedacted());
.map(mouse -> ManualRecategorization.builder()
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.type("published_information")
.annotationId(mouse.getId())
.fileId(TEST_FILE_ID)
.build())
.toList();
assertEquals("CBI.3.2", asyaLyon.getMatchedRule());
assertEquals("No vertebrate found", asyaLyon.getReason());
ManualRecategorization recategorization = ManualRecategorization.builder() ManualRecategorization recategorization = ManualRecategorization.builder()
.requestDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now())
@ -321,10 +307,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.build(); .build();
request.setManualRedactions(new ManualRedactions()); request.setManualRedactions(new ManualRedactions());
request.getManualRedactions() request.getManualRedactions().setRecategorizations(Set.of(recategorization));
.setRecategorizations(Stream.of(Stream.of(recategorization), recategorizations.stream()).flatMap(Function.identity()).collect(Collectors.toSet()));
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
RedactionLog redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); RedactionLog redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertFalse(redactionLog2.getRedactionLogEntry() assertFalse(redactionLog2.getRedactionLogEntry()
.stream() .stream()
@ -338,10 +324,14 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst() .findFirst()
.get(); .get();
for (RedactionLogEntry mouse : mouses) { var asyaLyon2 = redactionLog2.getRedactionLogEntry()
assertEquals("published_information", redactionLog2.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(mouse.getId())).findFirst().get().getType()); .stream()
} .filter(entry -> entry.getType().equals("CBI_author"))
.filter(entry -> entry.getValue().equals("Asya Lyon"))
.findFirst()
.get();
assertTrue(asyaLyon2.isRedacted());
assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size()); assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
} }

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.manualchanges; package com.iqser.red.service.redaction.v1.server.manualchanges;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.wildfly.common.Assert.assertFalse; import static org.wildfly.common.Assert.assertFalse;
import static org.wildfly.common.Assert.assertTrue; import static org.wildfly.common.Assert.assertTrue;
@ -53,6 +54,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
MockitoAnnotations.openMocks(this); MockitoAnnotations.openMocks(this);
when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f}); when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
when(dictionaryService.isHint(any(), any())).thenReturn(false);
} }
@ -88,7 +90,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
tempEntity.removeFromGraph(); tempEntity.removeFromGraph();
assertTrue(document.getEntities().isEmpty()); assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document); List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertTrue(notFoundManualEntities.isEmpty()); assertTrue(notFoundManualEntities.isEmpty());
assertEquals(1, document.getEntities().size()); assertEquals(1, document.getEntities().size());
} }
@ -116,7 +120,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
assertTrue(document.getEntities().isEmpty()); assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document); List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertEquals(1, notFoundManualEntities.size()); assertEquals(1, notFoundManualEntities.size());
assertTrue(document.getEntities().isEmpty()); assertTrue(document.getEntities().isEmpty());

27
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -423,7 +423,7 @@ rule "ETC.0.0: Purity Hint"
when when
$section: Section(containsStringIgnoreCase("purity")) $section: Section(containsStringIgnoreCase("purity"))
then then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section)
.forEach(hint -> hint.skip("ETC.0.0", "hint only")); .forEach(hint -> hint.skip("ETC.0.0", "hint only"));
end end
@ -667,7 +667,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then then
$entity.getIntersectingNodes().forEach(node -> update(node)); $entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -691,7 +691,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256 salience 256
when when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$entity.addEngines($recommendation.getEngines()); $entity.addEngines($recommendation.getEngines());
@ -704,7 +704,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256 salience 256
when when
$entity: TextEntity(entityType == EntityType.ENTITY, active()) $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
@ -713,17 +713,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6 // Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then then
$lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank); retract($lowerRank);
end end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------

8
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl
View File

@ -831,7 +831,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then then
$entity.getIntersectingNodes().forEach(node -> update(node)); $entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -855,7 +855,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256 salience 256
when when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$entity.addEngines($recommendation.getEngines()); $entity.addEngines($recommendation.getEngines());
@ -868,7 +868,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256 salience 256
when when
$entity: TextEntity(entityType == EntityType.ENTITY, active()) $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
@ -880,7 +880,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.7.0: remove all images" rule "X.7.0: remove all images"
salience 512 salience 512
when when
$image: Image(imageType != ImageType.OCR) $image: Image(imageType != ImageType.OCR, !hasManualChanges())
then then
$image.remove("X.7.0", "remove all images"); $image.remove("X.7.0", "remove all images");
retract($image); retract($image);

32
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl
View File

@ -74,8 +74,8 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
$section: Section(containsString("CTL/") || containsString("BL/")) $section: Section(containsString("CTL/") || containsString("BL/"))
then then
Stream.concat( Stream.concat(
entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section), entityCreationService.byString("CTL", "must_redact", EntityType.HINT, $section),
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) entityCreationService.byString("BL", "must_redact", EntityType.HINT, $section)
).forEach(entity -> entity.skip("SYN.0.0", "hint_only")); ).forEach(entity -> entity.skip("SYN.0.0", "hint_only"));
end end
@ -1012,7 +1012,7 @@ rule "ETC.0.0: Purity Hint"
when when
$section: Section(containsStringIgnoreCase("purity")) $section: Section(containsStringIgnoreCase("purity"))
then then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section)
.forEach(hint -> hint.skip("ETC.0.0", "hint only")); .forEach(hint -> hint.skip("ETC.0.0", "hint only"));
end end
@ -1340,7 +1340,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then then
$entity.getIntersectingNodes().forEach(node -> update(node)); $entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1364,7 +1364,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256 salience 256
when when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$entity.addEngines($recommendation.getEngines()); $entity.addEngines($recommendation.getEngines());
@ -1377,7 +1377,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256 salience 256
when when
$entity: TextEntity(entityType == EntityType.ENTITY, active()) $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
@ -1386,18 +1386,30 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6 // Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then then
$lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank); retract($lowerRank);
end end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

8
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -91,7 +91,7 @@ rule "H.2.0: Show headlines"
when when
$headline: Headline() $headline: Headline()
then then
entityCreationService.bySemanticNode($headline, "headline", EntityType.ENTITY); entityCreationService.bySemanticNode($headline, "headline", EntityType.HINT);
end end
@ -1232,7 +1232,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then then
$entity.getIntersectingNodes().forEach(node -> update(node)); $entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1256,7 +1256,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256 salience 256
when when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$entity.addEngines($recommendation.getEngines()); $entity.addEngines($recommendation.getEngines());
@ -1269,7 +1269,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256 salience 256
when when
$entity: TextEntity(entityType == EntityType.ENTITY, active()) $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");

29
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -74,8 +74,8 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
$section: Section(containsString("CTL/") || containsString("BL/")) $section: Section(containsString("CTL/") || containsString("BL/"))
then then
Stream.concat( Stream.concat(
entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section), entityCreationService.byString("CTL", "must_redact", EntityType.HINT, $section),
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) entityCreationService.byString("BL", "must_redact", EntityType.HINT, $section)
).forEach(entity -> entity.skip("SYN.0.0", "hint_only")); ).forEach(entity -> entity.skip("SYN.0.0", "hint_only"));
end end
@ -1051,7 +1051,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then then
$entity.getIntersectingNodes().forEach(node -> update(node)); $entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1075,7 +1075,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256 salience 256
when when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$entity.addEngines($recommendation.getEngines()); $entity.addEngines($recommendation.getEngines());
@ -1088,7 +1088,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256 salience 256
when when
$entity: TextEntity(entityType == EntityType.ENTITY, active()) $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
@ -1097,17 +1097,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6 // Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then then
$lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank); retract($lowerRank);
end end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------

35
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -236,7 +236,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type, $entityType: entityType, active()) $larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active()) $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained); retract($contained);
@ -248,7 +248,7 @@ rule "X.1.0: merge intersecting Entities of same type"
salience 64 salience 64
when when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active()) $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active()) $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type"); $first.remove("X.1.0", "merge intersecting Entities of same type");
@ -264,7 +264,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active()) $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then then
$entity.getIntersectingNodes().forEach(node -> update(node)); $entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -277,7 +277,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
salience 64 salience 64
when when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation); retract($recommendation);
@ -288,8 +288,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256 salience 256
when when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$entity.addEngines($recommendation.getEngines()); $entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
@ -301,8 +301,8 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256 salience 256
when when
$entity: TextEntity(entityType == EntityType.ENTITY, active()) $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation); retract($recommendation);
@ -310,17 +310,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6 // Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active()) $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then then
$lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank); retract($lowerRank);
end end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------