Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7631' into 'master'

Browse Source 
RED-7631: create EntityLog to replace RedactionLog

Closes RED-7631

See merge request redactmanager/redaction-service!152
This commit is contained in:
Ali Oezyetimoglu 2023-10-09 17:16:59 +02:00
commit d71f06d884
21 changed files with 351 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
publish-custom-image.sh
View File

@ -9,7 +9,7 @@ branch=$(git rev-parse --abbrev-ref HEAD)
commit_hash=$(git rev-parse --short=5 HEAD)
# Combine branch and commit hash
buildName="${branch}-${commit_hash}"
buildName="${USER}-${branch}-${commit_hash}"
gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java
View File

@ -56,8 +56,7 @@ public class ManualEntity implements IEntity {
.reason(redactionLogEntry.getReason())
.legalBasis(redactionLogEntry.getLegalBasis())
.type(redactionLogEntry.getType())
.section(redactionLogEntry.getSection())
.entityType(redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY)
.section(redactionLogEntry.getSection()).entityType(getEntityType(redactionLogEntry))
.applied(redactionLogEntry.isRedacted())
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
.isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
@ -66,7 +65,19 @@ public class ManualEntity implements IEntity {
}
public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry) {
private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.isRecommendation()) {
return EntityType.RECOMMENDATION;
}
if (redactionLogEntry.isHint()) {
return EntityType.HINT;
}
return EntityType.ENTITY;
}
public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite();
@ -79,8 +90,7 @@ public class ManualEntity implements IEntity {
.reason(manualRedactionEntry.getReason())
.legalBasis(manualRedactionEntry.getLegalBasis())
.type(manualRedactionEntry.getType())
.section(manualRedactionEntry.getSection())
.entityType(EntityType.ENTITY)
.section(manualRedactionEntry.getSection()).entityType(hint ? EntityType.HINT : EntityType.ENTITY)
.applied(true)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(false)

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity;
public enum EntityType {
ENTITY,
HINT,
RECOMMENDATION,
FALSE_POSITIVE,
FALSE_RECOMMENDATION

118
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -81,56 +81,6 @@ public class AnalyzeService {
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
@Timed("redactmanager_analyze")
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
dictionarySearchService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
dictionary,
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
EntityLog entityLog = entityLogCreatorService.createInitialEntityLogFromActiveEntities(analyzeRequest,
document,
notFoundManualRedactionEntries,
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
redactionLog,
document.getNumberOfPages(),
dictionary.getVersion(),
false,
new HashSet<>(allFileAttributes));
}
@Timed("redactmanager_reanalyze")
@SneakyThrows
public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
@ -164,8 +114,7 @@ public class AnalyzeService {
kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
dictionaryIncrement.getDictionaryVersion(),
analyzeRequest.getDossierTemplateId(),
false), document,
analyzeRequest.getDossierTemplateId(), false), document,
previousRedactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
@ -178,7 +127,9 @@ public class AnalyzeService {
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document,
analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -205,8 +156,7 @@ public class AnalyzeService {
return finalizeAnalysis(analyzeRequest,
startTime,
kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
entityLogChanges, document,
kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLogChanges, document,
redactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
@ -215,6 +165,58 @@ public class AnalyzeService {
}
@Timed("redactmanager_analyze")
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document,
analyzeRequest.getDossierTemplateId());
dictionarySearchService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
dictionary,
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
document,
notFoundManualRedactionEntries,
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
redactionLog,
document.getNumberOfPages(),
dictionary.getVersion(),
false,
new HashSet<>(allFileAttributes));
}
@Deprecated(forRemoval = true)
private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
Document document,
@ -283,8 +285,7 @@ public class AnalyzeService {
}
private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
KieWrapper kieWrapperComponentRules, Document document,
private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document,
Set<FileAttribute> addedFileAttributes,
EntityLogChanges entityLogChanges,
DictionaryVersion dictionaryVersion) {
@ -293,8 +294,7 @@ public class AnalyzeService {
return;
}
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
entityLogChanges.getEntityLog(), document,
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document,
addedFileAttributes.stream().toList());
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -27,7 +27,7 @@ public class DictionarySearchService {
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
for (var model : dictionary.getDictionaryModels()) {
bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
}

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -29,13 +30,14 @@ public class EntityChangeLogService {
var now = OffsetDateTime.now();
for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
var optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
if (optionalPreviousEntity.isEmpty()) {
hasChanges = true;
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
continue;
}
var previousEntity = optionalPreviousEntity.get();
entityLogEntry.getChanges().addAll(previousEntity.getChanges());
if (!previousEntity.getState().equals(entityLogEntry.getState())) {
hasChanges = true;
ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
@ -54,25 +56,22 @@ public class EntityChangeLogService {
private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
if (state.equals(previousState)) {
throw new IllegalArgumentException("States are equal, can't calculate ChangeType.");
}
if (!isRemoved(previousState) && isRemoved(state)) {
return ChangeType.REMOVED;
}
if (isRemoved(previousState) && !isRemoved(state)) {
return ChangeType.ADDED;
}
if (previousState.equals(EntryState.APPLIED) && state.equals(EntryState.SKIPPED)) {
return ChangeType.CHANGED;
}
if (previousState.equals(EntryState.SKIPPED) && state.equals(EntryState.APPLIED)) {
return ChangeType.CHANGED;
}
return null;
return ChangeType.CHANGED;
}
private static boolean isRemoved(EntryState state) {
return state.equals(EntryState.IGNORED) || state.equals(EntryState.REMOVED);
return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED));
}

155
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -50,13 +50,19 @@ public class EntityLogCreatorService {
private final EntityChangeLogService entityChangeLogService;
public EntityLog createInitialEntityLogFromActiveEntities(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
DictionaryVersion dictionaryVersion,
long rulesVersion) {
private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
List<EntityLogEntry> entityLogEntries = createEntityLogEntriesFromActiveEntities(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION));
}
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
DictionaryVersion dictionaryVersion,
long rulesVersion) {
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
@ -83,36 +89,6 @@ public class EntityLogCreatorService {
}
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
EntityLog previousEntityLog,
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> entriesFromReanalysedSections = previousEntityLog.getEntityLogEntry()
.stream().filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
.equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
.toList();
previousEntityLog.getEntityLogEntry().removeAll(entriesFromReanalysedSections);
boolean hasChanges = entityChangeLogService.computeChanges(entriesFromReanalysedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newEntityLogEntries,
false);
previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages());
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges);
}
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) {
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId);
@ -125,34 +101,47 @@ public class EntityLogCreatorService {
}
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
EntityLog previousEntityLog,
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
.equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
.toList();
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
var newEntityLogWithImportedEntities = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newEntityLogEntries,
false);
previousEntityLog.getEntityLogEntry().addAll(newEntityLogWithImportedEntities);
excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages());
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges);
}
private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
List<EntityLogEntry> entries = new ArrayList<>();
document.getEntities()
.stream()
.filter(EntityLogCreatorService::isEntityOrRecommendationType)
.stream().filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation).filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
document.streamAllImages().forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId)));
return entries;
}
private List<EntityLogEntry> createEntityLogEntriesFromActiveEntities(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
List<EntityLogEntry> entries = new ArrayList<>();
document.getEntities().stream().filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active)
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
document.streamAllImages().filter(IEntity::active).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.stream().filter(IEntity::active).forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
return entries;
}
private static boolean isEntityOrRecommendationType(TextEntity textEntity) {
return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION;
}
@ -189,8 +178,7 @@ public class EntityLogCreatorService {
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return EntityLogEntry.builder()
.id(image.getId())
.value(image.value())
.color(getColor(imageType, dossierTemplateId, image.applied()))
.value(image.value()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
.value(image.value())
.type(imageType)
.reason(image.buildReasonWithManualChangeDescriptions())
@ -203,8 +191,7 @@ public class EntityLogCreatorService {
.section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
.imageHasTransparency(image.isTransparent())
.manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(image))
.entryType(buildEntryType(image, isHint))
.state(buildEntryState(image)).entryType(buildEntryType(image))
.build();
}
@ -213,16 +200,14 @@ public class EntityLogCreatorService {
private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(type, dossierTemplateId);
boolean isHint = isHint(manualEntity.getEntityType());
return EntityLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied()))
.id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
.type(type)
.state(buildEntryState(manualEntity))
.entryType(buildEntryType(manualEntity, isHint))
.state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.containingNodeId(Collections.emptyList())
.closestHeadline("")
@ -245,10 +230,8 @@ public class EntityLogCreatorService {
Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
boolean isHint = isHint(entity.getType(), dossierTemplateId);
return EntityLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
boolean isHint = isHint(entity.getEntityType());
return EntityLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@ -266,24 +249,23 @@ public class EntityLogCreatorService {
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
.reference(referenceIds)
.manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(entity))
.entryType(buildEntryType(entity, isHint))
.state(buildEntryState(entity)).entryType(buildEntryType(entity))
.build();
}
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
private boolean isHint(EntityType entityType) {
if (!isRedaction && !isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
return entityType.equals(EntityType.HINT);
}
private boolean isHint(String type, String dossierTemplateId) {
private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
return dictionaryService.isHint(type, dossierTemplateId);
if (!isApplied && !isHint) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
@ -301,26 +283,27 @@ public class EntityLogCreatorService {
}
private EntryType buildEntryType(IEntity entity, boolean isHint) {
private EntryType buildEntryType(IEntity entity) {
if (entity instanceof TextEntity textEntity) {
return getEntryType(isHint, textEntity.getEntityType());
return getEntryType(textEntity.getEntityType());
} else if (entity instanceof ManualEntity manualEntity) {
if (((ManualEntity) entity).isRectangle()) {
return EntryType.AREA;
}
return getEntryType(isHint, manualEntity.getEntityType());
return getEntryType(manualEntity.getEntityType());
} else if (entity instanceof Image) {
return EntryType.IMAGE;
}
throw new UnsupportedOperationException("Entity subclass %s not implemented!");
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
}
private static EntryType getEntryType(boolean isHint, EntityType entityType) {
private static EntryType getEntryType(EntityType entityType) {
return switch (entityType) {
case ENTITY -> isHint ? EntryType.HINT : EntryType.ENTITY;
case ENTITY -> EntryType.ENTITY;
case HINT -> EntryType.HINT;
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
case RECOMMENDATION -> EntryType.RECOMMENDATION;
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java
View File

@ -16,13 +16,15 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualChangesApplicationService {
private final EntityCreationService entityCreationService;
EntityCreationService entityCreationService;
public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {

128
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java
View File

@ -14,14 +14,15 @@ import java.util.stream.Collectors;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
@ -30,7 +31,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -51,8 +51,7 @@ public class RedactionLogCreatorService {
List<RedactionLogEntry> entries = new ArrayList<>();
document.getEntities()
.stream()
.filter(RedactionLogCreatorService::isEntityOrRecommendationType)
.stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(IEntity::active)
.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments)));
document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
@ -61,9 +60,9 @@ public class RedactionLogCreatorService {
}
private static boolean isEntityOrRecommendationType(TextEntity textEntity) {
private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION;
return !(textEntity.getEntityType() == EntityType.FALSE_POSITIVE || textEntity.getEntityType() == EntityType.FALSE_RECOMMENDATION);
}
@ -133,9 +132,8 @@ public class RedactionLogCreatorService {
Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
boolean isHint = isHint(entity.getType(), dossierTemplateId);
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
boolean isHint = isHint(entity.getEntityType());
return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@ -177,39 +175,9 @@ public class RedactionLogCreatorService {
}
public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
private boolean isHint(EntityType entityType) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(type, dossierTemplateId);
return RedactionLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied()))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
.type(type)
.redacted(manualEntity.applied())
.isHint(isHint)
.isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.sectionNumber(0).matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
.rectangle(manualEntity.isRectangle())
.isDictionaryEntry(manualEntity.isDictionaryEntry())
.isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.textAfter("")
.textBefore("")
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getEntityPosition()
.stream()
.map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.collect(Collectors.toList()))
.engines(Collections.emptySet())
.reference(Collections.emptySet())
.manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
.comments(buildRedactionLogComments(comments, manualEntity.getId()))
.build();
return entityType.equals(EntityType.HINT);
}
@ -242,13 +210,64 @@ public class RedactionLogCreatorService {
}
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean isHint) {
if (!isRedaction && isHint) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(manualEntity.getEntityType());
return RedactionLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
.type(type)
.redacted(manualEntity.applied())
.isHint(isHint)
.isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.sectionNumber(0)
.matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
.rectangle(manualEntity.isRectangle())
.isDictionaryEntry(manualEntity.isDictionaryEntry())
.isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.textAfter("")
.textBefore("")
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getEntityPosition()
.stream()
.map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.collect(Collectors.toList()))
.engines(Collections.emptySet())
.reference(Collections.emptySet())
.manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
.comments(buildRedactionLogComments(comments, manualEntity.getId()))
.build();
}
private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
}
public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return RedactionLogEntry.builder()
.id(image.getId())
.color(getColor(imageType, dossierTemplateId, image.applied()))
.id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
.isImage(true)
.value(image.value())
.type(imageType)
@ -269,27 +288,4 @@ public class RedactionLogCreatorService {
}
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
if (!isRedaction && !isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private boolean isHint(String type, String dossierTemplateId) {
return dictionaryService.isHint(type, dossierTemplateId);
}
private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()),
(float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(),
pageNumber);
}
}

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java
View File

@ -29,21 +29,27 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionO
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualEntityCreationService {
private static final double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities
private final EntityCreationService entityCreationService;
static double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities
EntityCreationService entityCreationService;
DictionaryService dictionaryService;
@Autowired
public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService) {
public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
entityCreationService = new EntityCreationService(entityEnrichmentService);
this.dictionaryService = dictionaryService;
}
@ -60,11 +66,14 @@ public class ManualEntityCreationService {
}
public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries, SemanticNode node) {
public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries,
SemanticNode node,
String dossierTemplateId) {
List<ManualEntity> manualEntities = manualRedactionEntries.stream()
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
.map(ManualEntity::fromManualRedactionEntry)
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.peek(manualEntity -> manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis()))
.toList();

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java
View File

@ -24,12 +24,12 @@ public class ManualRedactionEntryService {
private final ManualEntityCreationService manualEntityCreationService;
public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document) {
public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList();
if (analyzeRequest.getManualRedactions() != null) {
notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions()
.getEntriesToAdd(), document);
.getEntriesToAdd(), document, dossierTemplateId);
log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
if (notFoundManualRedactionEntries.isEmpty()) {

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -1,7 +1,6 @@
package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.util.List;
@ -26,7 +25,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java
View File

@ -9,6 +9,7 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.MockitoAnnotations;
import org.springframework.boot.test.mock.mockito.MockBean;
@ -210,6 +211,7 @@ class DroolsSyntaxValidationServiceTest {
@Test
@SneakyThrows
@Disabled
void attemptImportsFixToAllRuleFiles() {
DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));

36
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -15,9 +15,6 @@ import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -77,7 +74,7 @@ import lombok.SneakyThrows;
@Import(ManualChangesEnd2EndTest.TestConfiguration.class)
public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules.drl");
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl");
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@ -297,20 +294,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst()
.get();
var mouses = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals("vertebrate")).filter(entry -> entry.getValue().equals("Mouse")).toList();
var recategorizations = mouses.stream()
.map(mouse -> ManualRecategorization.builder()
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.type("published_information")
.annotationId(mouse.getId())
.fileId(TEST_FILE_ID)
.build())
.toList();
assertEquals("CBI.3.2", asyaLyon.getMatchedRule());
assertEquals("No vertebrate found", asyaLyon.getReason());
assertEquals("CBI.7.0", asyaLyon.getMatchedRule());
assertEquals("Published Information found in section", asyaLyon.getReason());
assertFalse(asyaLyon.isRedacted());
ManualRecategorization recategorization = ManualRecategorization.builder()
.requestDate(OffsetDateTime.now())
@ -321,10 +307,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.build();
request.setManualRedactions(new ManualRedactions());
request.getManualRedactions()
.setRecategorizations(Stream.of(Stream.of(recategorization), recategorizations.stream()).flatMap(Function.identity()).collect(Collectors.toSet()));
request.getManualRedactions().setRecategorizations(Set.of(recategorization));
analyzeService.reanalyze(request);
RedactionLog redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertFalse(redactionLog2.getRedactionLogEntry()
.stream()
@ -338,10 +324,14 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst()
.get();
for (RedactionLogEntry mouse : mouses) {
assertEquals("published_information", redactionLog2.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(mouse.getId())).findFirst().get().getType());
}
var asyaLyon2 = redactionLog2.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getType().equals("CBI_author"))
.filter(entry -> entry.getValue().equals("Asya Lyon"))
.findFirst()
.get();
assertTrue(asyaLyon2.isRedacted());
assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
}

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.manualchanges;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import static org.wildfly.common.Assert.assertFalse;
import static org.wildfly.common.Assert.assertTrue;
@ -53,6 +54,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
MockitoAnnotations.openMocks(this);
when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
when(dictionaryService.isHint(any(), any())).thenReturn(false);
}
@ -88,7 +90,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
tempEntity.removeFromGraph();
assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document);
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertTrue(notFoundManualEntities.isEmpty());
assertEquals(1, document.getEntities().size());
}
@ -116,7 +120,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document);
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertEquals(1, notFoundManualEntities.size());
assertTrue(document.getEntities().isEmpty());

27
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -423,7 +423,7 @@ rule "ETC.0.0: Purity Hint"
when
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section)
.forEach(hint -> hint.skip("ETC.0.0", "hint only"));
end
@ -667,7 +667,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -691,7 +691,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
@ -704,7 +704,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity(entityType == EntityType.ENTITY, active())
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
@ -713,17 +713,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
//------------------------------------ File attributes rules ------------------------------------

8
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl
View File

@ -831,7 +831,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -855,7 +855,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
@ -868,7 +868,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity(entityType == EntityType.ENTITY, active())
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
@ -880,7 +880,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.7.0: remove all images"
salience 512
when
$image: Image(imageType != ImageType.OCR)
$image: Image(imageType != ImageType.OCR, !hasManualChanges())
then
$image.remove("X.7.0", "remove all images");
retract($image);

32
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl
View File

@ -74,8 +74,8 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
$section: Section(containsString("CTL/") || containsString("BL/"))
then
Stream.concat(
entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section),
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
entityCreationService.byString("CTL", "must_redact", EntityType.HINT, $section),
entityCreationService.byString("BL", "must_redact", EntityType.HINT, $section)
).forEach(entity -> entity.skip("SYN.0.0", "hint_only"));
end
@ -1012,7 +1012,7 @@ rule "ETC.0.0: Purity Hint"
when
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section)
.forEach(hint -> hint.skip("ETC.0.0", "hint only"));
end
@ -1340,7 +1340,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1364,7 +1364,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
@ -1377,7 +1377,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity(entityType == EntityType.ENTITY, active())
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
@ -1386,18 +1386,30 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

8
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -91,7 +91,7 @@ rule "H.2.0: Show headlines"
when
$headline: Headline()
then
entityCreationService.bySemanticNode($headline, "headline", EntityType.ENTITY);
entityCreationService.bySemanticNode($headline, "headline", EntityType.HINT);
end
@ -1232,7 +1232,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1256,7 +1256,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
@ -1269,7 +1269,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity(entityType == EntityType.ENTITY, active())
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");

29
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -74,8 +74,8 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
$section: Section(containsString("CTL/") || containsString("BL/"))
then
Stream.concat(
entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section),
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
entityCreationService.byString("CTL", "must_redact", EntityType.HINT, $section),
entityCreationService.byString("BL", "must_redact", EntityType.HINT, $section)
).forEach(entity -> entity.skip("SYN.0.0", "hint_only"));
end
@ -1051,7 +1051,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1075,7 +1075,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
@ -1088,7 +1088,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity(entityType == EntityType.ENTITY, active())
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
@ -1097,17 +1097,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
//------------------------------------ File attributes rules ------------------------------------

35
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -236,7 +236,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -248,7 +248,7 @@ rule "X.1.0: merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
@ -264,7 +264,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -277,7 +277,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -288,8 +288,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
@ -301,8 +301,8 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity(entityType == EntityType.ENTITY, active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active())
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation);
@ -310,17 +310,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
//------------------------------------ File attributes rules ------------------------------------