diff --git a/publish-custom-image.sh b/publish-custom-image.sh index d61a8dc5..89b9f8d9 100755 --- a/publish-custom-image.sh +++ b/publish-custom-image.sh @@ -9,7 +9,7 @@ branch=$(git rev-parse --abbrev-ref HEAD) commit_hash=$(git rev-parse --short=5 HEAD) # Combine branch and commit hash -buildName="${branch}-${commit_hash}" +buildName="${USER}-${branch}-${commit_hash}" gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java index e3104457..a080238c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java @@ -56,8 +56,7 @@ public class ManualEntity implements IEntity { .reason(redactionLogEntry.getReason()) .legalBasis(redactionLogEntry.getLegalBasis()) .type(redactionLogEntry.getType()) - .section(redactionLogEntry.getSection()) - .entityType(redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY) + .section(redactionLogEntry.getSection()).entityType(getEntityType(redactionLogEntry)) .applied(redactionLogEntry.isRedacted()) .isDictionaryEntry(redactionLogEntry.isDictionaryEntry()) .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry()) @@ -66,7 +65,19 @@ public class ManualEntity implements IEntity { } - public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry) { + private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) { + + if (redactionLogEntry.isRecommendation()) { + return EntityType.RECOMMENDATION; + } + if (redactionLogEntry.isHint()) { + return EntityType.HINT; + } + return EntityType.ENTITY; + } + + + public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) { List rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList(); ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(); @@ -79,8 +90,7 @@ public class ManualEntity implements IEntity { .reason(manualRedactionEntry.getReason()) .legalBasis(manualRedactionEntry.getLegalBasis()) .type(manualRedactionEntry.getType()) - .section(manualRedactionEntry.getSection()) - .entityType(EntityType.ENTITY) + .section(manualRedactionEntry.getSection()).entityType(hint ? EntityType.HINT : EntityType.ENTITY) .applied(true) .isDictionaryEntry(false) .isDossierDictionaryEntry(false) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java index b86bc323..6fb2d781 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity; public enum EntityType { ENTITY, + HINT, RECOMMENDATION, FALSE_POSITIVE, FALSE_RECOMMENDATION diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 7fbff336..6abf510e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -81,56 +81,6 @@ public class AnalyzeService { FunctionTimerValues redactmanagerAnalyzePagewiseValues; - @Timed("redactmanager_analyze") - public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { - - long startTime = System.currentTimeMillis(); - - var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); - log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT); - log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); - log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document); - - dictionarySearchService.addDictionaryEntities(dictionary, document); - log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), - document, - dictionary, - analyzeRequest.getFileAttributes(), - analyzeRequest.getManualRedactions(), - nerEntities); - log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); - EntityLog entityLog = entityLogCreatorService.createInitialEntityLogFromActiveEntities(analyzeRequest, - document, - notFoundManualRedactionEntries, - dictionary.getVersion(), - kieWrapperEntityRules.rulesVersion()); - - return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document, - redactionLog, - document.getNumberOfPages(), - dictionary.getVersion(), - false, - new HashSet<>(allFileAttributes)); - } - - @Timed("redactmanager_reanalyze") @SneakyThrows public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { @@ -164,8 +114,7 @@ public class AnalyzeService { kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog, dictionaryIncrement.getDictionaryVersion(), - analyzeRequest.getDossierTemplateId(), - false), document, + analyzeRequest.getDossierTemplateId(), false), document, previousRedactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -178,7 +127,9 @@ public class AnalyzeService { NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document); + List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, + document, + analyzeRequest.getDossierTemplateId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -205,8 +156,7 @@ public class AnalyzeService { return finalizeAnalysis(analyzeRequest, startTime, - kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), - entityLogChanges, document, + kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLogChanges, document, redactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -215,6 +165,58 @@ public class AnalyzeService { } + @Timed("redactmanager_analyze") + public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { + + long startTime = System.currentTimeMillis(); + + var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); + log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT); + log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); + log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, + document, + analyzeRequest.getDossierTemplateId()); + + dictionarySearchService.addDictionaryEntities(dictionary, document); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), + document, + dictionary, + analyzeRequest.getFileAttributes(), + analyzeRequest.getManualRedactions(), + nerEntities); + log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); + EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest, + document, + notFoundManualRedactionEntries, + dictionary.getVersion(), + kieWrapperEntityRules.rulesVersion()); + + return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document, + redactionLog, + document.getNumberOfPages(), + dictionary.getVersion(), + false, + new HashSet<>(allFileAttributes)); + } + + @Deprecated(forRemoval = true) private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest, Document document, @@ -283,8 +285,7 @@ public class AnalyzeService { } - private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, - KieWrapper kieWrapperComponentRules, Document document, + private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document, Set addedFileAttributes, EntityLogChanges entityLogChanges, DictionaryVersion dictionaryVersion) { @@ -293,8 +294,7 @@ public class AnalyzeService { return; } - List components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), - entityLogChanges.getEntityLog(), document, + List components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document, addedFileAttributes.stream().toList()); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 0b07ed82..16f6b5df 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -27,7 +27,7 @@ public class DictionarySearchService { public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { for (var model : dictionary.getDictionaryModels()) { - bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index ed49f10d..039b9290 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service; import java.time.OffsetDateTime; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -29,13 +30,14 @@ public class EntityChangeLogService { var now = OffsetDateTime.now(); for (EntityLogEntry entityLogEntry : newEntityLogEntries) { - var optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny(); + Optional optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny(); if (optionalPreviousEntity.isEmpty()) { hasChanges = true; entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)); continue; } var previousEntity = optionalPreviousEntity.get(); + entityLogEntry.getChanges().addAll(previousEntity.getChanges()); if (!previousEntity.getState().equals(entityLogEntry.getState())) { hasChanges = true; ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState()); @@ -54,25 +56,22 @@ public class EntityChangeLogService { private ChangeType calculateChangeType(EntryState state, EntryState previousState) { + if (state.equals(previousState)) { + throw new IllegalArgumentException("States are equal, can't calculate ChangeType."); + } if (!isRemoved(previousState) && isRemoved(state)) { return ChangeType.REMOVED; } if (isRemoved(previousState) && !isRemoved(state)) { return ChangeType.ADDED; } - if (previousState.equals(EntryState.APPLIED) && state.equals(EntryState.SKIPPED)) { - return ChangeType.CHANGED; - } - if (previousState.equals(EntryState.SKIPPED) && state.equals(EntryState.APPLIED)) { - return ChangeType.CHANGED; - } - return null; + return ChangeType.CHANGED; } private static boolean isRemoved(EntryState state) { - return state.equals(EntryState.IGNORED) || state.equals(EntryState.REMOVED); + return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 5c32c3d4..cb517347 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -50,13 +50,19 @@ public class EntityLogCreatorService { private final EntityChangeLogService entityChangeLogService; - public EntityLog createInitialEntityLogFromActiveEntities(AnalyzeRequest analyzeRequest, - Document document, - List notFoundManualRedactionEntries, - DictionaryVersion dictionaryVersion, - long rulesVersion) { + private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) { - List entityLogEntries = createEntityLogEntriesFromActiveEntities(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)); + } + + + public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + DictionaryVersion dictionaryVersion, + long rulesVersion) { + + List entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); @@ -83,36 +89,6 @@ public class EntityLogCreatorService { } - public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, - Document document, - List notFoundManualRedactionEntries, - EntityLog previousEntityLog, - Set sectionsToReanalyseIds, - DictionaryVersion dictionaryVersion) { - - List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); - Set newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); - List entriesFromReanalysedSections = previousEntityLog.getEntityLogEntry() - .stream().filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType() - .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)) - .toList(); - previousEntityLog.getEntityLogEntry().removeAll(entriesFromReanalysedSections); - boolean hasChanges = entityChangeLogService.computeChanges(entriesFromReanalysedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); - - var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - newEntityLogEntries, - false); - - previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries); - - excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages()); - - return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges); - } - - public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) { List legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId); @@ -125,34 +101,47 @@ public class EntityLogCreatorService { } + public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + EntityLog previousEntityLog, + Set sectionsToReanalyseIds, + DictionaryVersion dictionaryVersion) { + + List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + Set newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); + List previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType() + .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)) + .toList(); + previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections); + + boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); + + var newEntityLogWithImportedEntities = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + newEntityLogEntries, + false); + + previousEntityLog.getEntityLogEntry().addAll(newEntityLogWithImportedEntities); + + excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages()); + + return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges); + } + + private List createEntityLogEntries(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { List entries = new ArrayList<>(); document.getEntities() - .stream() - .filter(EntityLogCreatorService::isEntityOrRecommendationType) + .stream().filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation).filter(entity -> !entity.removed()) .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); - document.streamAllImages().forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); - notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); + document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); + notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId))); return entries; - - } - - - private List createEntityLogEntriesFromActiveEntities(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { - - List entries = new ArrayList<>(); - document.getEntities().stream().filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active) - .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); - document.streamAllImages().filter(IEntity::active).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); - notFoundManualRedactionEntries.stream().filter(IEntity::active).forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); - return entries; - } - - - private static boolean isEntityOrRecommendationType(TextEntity textEntity) { - - return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION; } @@ -189,8 +178,7 @@ public class EntityLogCreatorService { boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); return EntityLogEntry.builder() .id(image.getId()) - .value(image.value()) - .color(getColor(imageType, dossierTemplateId, image.applied())) + .value(image.value()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint)) .value(image.value()) .type(imageType) .reason(image.buildReasonWithManualChangeDescriptions()) @@ -203,8 +191,7 @@ public class EntityLogCreatorService { .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) .imageHasTransparency(image.isTransparent()) .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) - .state(buildEntryState(image)) - .entryType(buildEntryType(image, isHint)) + .state(buildEntryState(image)).entryType(buildEntryType(image)) .build(); } @@ -213,16 +200,14 @@ public class EntityLogCreatorService { private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) { String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); - boolean isHint = isHint(type, dossierTemplateId); + boolean isHint = isHint(manualEntity.getEntityType()); return EntityLogEntry.builder() - .id(manualEntity.getId()) - .color(getColor(type, dossierTemplateId, manualEntity.applied())) + .id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint)) .reason(manualEntity.buildReasonWithManualChangeDescriptions()) .legalBasis(manualEntity.legalBasis()) .value(manualEntity.value()) .type(type) - .state(buildEntryState(manualEntity)) - .entryType(buildEntryType(manualEntity, isHint)) + .state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity)) .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) .containingNodeId(Collections.emptyList()) .closestHeadline("") @@ -245,10 +230,8 @@ public class EntityLogCreatorService { Set referenceIds = new HashSet<>(); entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); - int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); - boolean isHint = isHint(entity.getType(), dossierTemplateId); - return EntityLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, entity.applied())) + boolean isHint = isHint(entity.getEntityType()); + return EntityLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) @@ -266,24 +249,23 @@ public class EntityLogCreatorService { .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) .reference(referenceIds) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) - .state(buildEntryState(entity)) - .entryType(buildEntryType(entity, isHint)) + .state(buildEntryState(entity)).entryType(buildEntryType(entity)) .build(); } - private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { + private boolean isHint(EntityType entityType) { - if (!isRedaction && !isHint(type, dossierTemplateId)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); + return entityType.equals(EntityType.HINT); } - private boolean isHint(String type, String dossierTemplateId) { + private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) { - return dictionaryService.isHint(type, dossierTemplateId); + if (!isApplied && !isHint) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); } @@ -301,26 +283,27 @@ public class EntityLogCreatorService { } - private EntryType buildEntryType(IEntity entity, boolean isHint) { + private EntryType buildEntryType(IEntity entity) { if (entity instanceof TextEntity textEntity) { - return getEntryType(isHint, textEntity.getEntityType()); + return getEntryType(textEntity.getEntityType()); } else if (entity instanceof ManualEntity manualEntity) { if (((ManualEntity) entity).isRectangle()) { return EntryType.AREA; } - return getEntryType(isHint, manualEntity.getEntityType()); + return getEntryType(manualEntity.getEntityType()); } else if (entity instanceof Image) { return EntryType.IMAGE; } - throw new UnsupportedOperationException("Entity subclass %s not implemented!"); + throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass())); } - private static EntryType getEntryType(boolean isHint, EntityType entityType) { + private static EntryType getEntryType(EntityType entityType) { return switch (entityType) { - case ENTITY -> isHint ? EntryType.HINT : EntryType.ENTITY; + case ENTITY -> EntryType.ENTITY; + case HINT -> EntryType.HINT; case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; case RECOMMENDATION -> EntryType.RECOMMENDATION; case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index 336c4092..2df1434e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -16,13 +16,15 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ManualChangesApplicationService { - private final EntityCreationService entityCreationService; - + EntityCreationService entityCreationService; public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index 32928912..7c98a09d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -14,14 +14,15 @@ import java.util.stream.Collectors; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; @@ -30,7 +31,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -51,8 +51,7 @@ public class RedactionLogCreatorService { List entries = new ArrayList<>(); document.getEntities() - .stream() - .filter(RedactionLogCreatorService::isEntityOrRecommendationType) + .stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation) .filter(IEntity::active) .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments))); document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments))); @@ -61,9 +60,9 @@ public class RedactionLogCreatorService { } - private static boolean isEntityOrRecommendationType(TextEntity textEntity) { + private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) { - return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION; + return !(textEntity.getEntityType() == EntityType.FALSE_POSITIVE || textEntity.getEntityType() == EntityType.FALSE_RECOMMENDATION); } @@ -133,9 +132,8 @@ public class RedactionLogCreatorService { Set referenceIds = new HashSet<>(); entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); - boolean isHint = isHint(entity.getType(), dossierTemplateId); - return RedactionLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, entity.applied())) + boolean isHint = isHint(entity.getEntityType()); + return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) @@ -177,39 +175,9 @@ public class RedactionLogCreatorService { } - public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map> comments) { + private boolean isHint(EntityType entityType) { - String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); - boolean isHint = isHint(type, dossierTemplateId); - return RedactionLogEntry.builder() - .id(manualEntity.getId()) - .color(getColor(type, dossierTemplateId, manualEntity.applied())) - .reason(manualEntity.buildReasonWithManualChangeDescriptions()) - .legalBasis(manualEntity.legalBasis()) - .value(manualEntity.value()) - .type(type) - .redacted(manualEntity.applied()) - .isHint(isHint) - .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION)) - .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) - .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) - .sectionNumber(0).matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) - .rectangle(manualEntity.isRectangle()) - .isDictionaryEntry(manualEntity.isDictionaryEntry()) - .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) - .textAfter("") - .textBefore("") - .startOffset(-1) - .endOffset(-1) - .positions(manualEntity.getEntityPosition() - .stream() - .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())) - .collect(Collectors.toList())) - .engines(Collections.emptySet()) - .reference(Collections.emptySet()) - .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint)) - .comments(buildRedactionLogComments(comments, manualEntity.getId())) - .build(); + return entityType.equals(EntityType.HINT); } @@ -242,13 +210,64 @@ public class RedactionLogCreatorService { } + private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean isHint) { + + if (!isRedaction && isHint) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); + } + + + public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map> comments) { + + String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); + boolean isHint = isHint(manualEntity.getEntityType()); + return RedactionLogEntry.builder() + .id(manualEntity.getId()) + .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint)) + .reason(manualEntity.buildReasonWithManualChangeDescriptions()) + .legalBasis(manualEntity.legalBasis()) + .value(manualEntity.value()) + .type(type) + .redacted(manualEntity.applied()) + .isHint(isHint) + .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION)) + .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) + .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) + .sectionNumber(0) + .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) + .rectangle(manualEntity.isRectangle()) + .isDictionaryEntry(manualEntity.isDictionaryEntry()) + .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) + .textAfter("") + .textBefore("") + .startOffset(-1) + .endOffset(-1) + .positions(manualEntity.getEntityPosition() + .stream() + .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .collect(Collectors.toList())) + .engines(Collections.emptySet()) + .reference(Collections.emptySet()) + .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint)) + .comments(buildRedactionLogComments(comments, manualEntity.getId())) + .build(); + } + + + private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) { + + return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber); + } + + public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map> comments) { String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); return RedactionLogEntry.builder() - .id(image.getId()) - .color(getColor(imageType, dossierTemplateId, image.applied())) + .id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint)) .isImage(true) .value(image.value()) .type(imageType) @@ -269,27 +288,4 @@ public class RedactionLogCreatorService { } - - private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { - - if (!isRedaction && !isHint(type, dossierTemplateId)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private boolean isHint(String type, String dossierTemplateId) { - - return dictionaryService.isHint(type, dossierTemplateId); - } - - - private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) { - - return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), - (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), - pageNumber); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java index 0fd6373f..f2f6cb70 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java @@ -29,21 +29,27 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionO import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ManualEntityCreationService { - private static final double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities - private final EntityCreationService entityCreationService; + static double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities + EntityCreationService entityCreationService; + DictionaryService dictionaryService; @Autowired - public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService) { + public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) { entityCreationService = new EntityCreationService(entityEnrichmentService); + this.dictionaryService = dictionaryService; } @@ -60,11 +66,14 @@ public class ManualEntityCreationService { } - public List createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set manualRedactionEntries, SemanticNode node) { + public List createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set manualRedactionEntries, + SemanticNode node, + String dossierTemplateId) { List manualEntities = manualRedactionEntries.stream() .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary())) - .map(ManualEntity::fromManualRedactionEntry) + .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry, + dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) .peek(manualEntity -> manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis())) .toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java index 366acff3..86902bff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java @@ -24,12 +24,12 @@ public class ManualRedactionEntryService { private final ManualEntityCreationService manualEntityCreationService; - public List addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document) { + public List addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) { List notFoundManualRedactionEntries = Collections.emptyList(); if (analyzeRequest.getManualRedactions() != null) { notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions() - .getEntriesToAdd(), document); + .getEntriesToAdd(), document, dossierTemplateId); log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } if (notFoundManualRedactionEntries.isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 2a93a20c..45909ba4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -1,7 +1,6 @@ package com.iqser.red.service.redaction.v1.server; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.util.List; @@ -26,7 +25,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java index 4da99ea9..ee87df62 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java @@ -9,6 +9,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.MockitoAnnotations; import org.springframework.boot.test.mock.mockito.MockBean; @@ -210,6 +211,7 @@ class DroolsSyntaxValidationServiceTest { @Test @SneakyThrows + @Disabled void attemptImportsFixToAllRuleFiles() { DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index c51cadd6..04cc9b95 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -15,9 +15,6 @@ import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.UUID; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -77,7 +74,7 @@ import lombok.SneakyThrows; @Import(ManualChangesEnd2EndTest.TestConfiguration.class) public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { - private static final String RULES = loadFromClassPath("drools/rules.drl"); + private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl"); private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl"); @Autowired private EntityEnrichmentService entityEnrichmentService; @@ -297,20 +294,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .findFirst() .get(); - var mouses = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals("vertebrate")).filter(entry -> entry.getValue().equals("Mouse")).toList(); - - var recategorizations = mouses.stream() - .map(mouse -> ManualRecategorization.builder() - .requestDate(OffsetDateTime.now()) - .status(AnnotationStatus.APPROVED) - .type("published_information") - .annotationId(mouse.getId()) - .fileId(TEST_FILE_ID) - .build()) - .toList(); - - assertEquals("CBI.3.2", asyaLyon.getMatchedRule()); - assertEquals("No vertebrate found", asyaLyon.getReason()); + assertEquals("CBI.7.0", asyaLyon.getMatchedRule()); + assertEquals("Published Information found in section", asyaLyon.getReason()); + assertFalse(asyaLyon.isRedacted()); ManualRecategorization recategorization = ManualRecategorization.builder() .requestDate(OffsetDateTime.now()) @@ -321,10 +307,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .build(); request.setManualRedactions(new ManualRedactions()); - request.getManualRedactions() - .setRecategorizations(Stream.of(Stream.of(recategorization), recategorizations.stream()).flatMap(Function.identity()).collect(Collectors.toSet())); + request.getManualRedactions().setRecategorizations(Set.of(recategorization)); analyzeService.reanalyze(request); + RedactionLog redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); assertFalse(redactionLog2.getRedactionLogEntry() .stream() @@ -338,10 +324,14 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .findFirst() .get(); - for (RedactionLogEntry mouse : mouses) { - assertEquals("published_information", redactionLog2.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(mouse.getId())).findFirst().get().getType()); - } + var asyaLyon2 = redactionLog2.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getType().equals("CBI_author")) + .filter(entry -> entry.getValue().equals("Asya Lyon")) + .findFirst() + .get(); + assertTrue(asyaLyon2.isRedacted()); assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java index d33f2724..4366b958 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.manualchanges; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.wildfly.common.Assert.assertFalse; import static org.wildfly.common.Assert.assertTrue; @@ -53,6 +54,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes MockitoAnnotations.openMocks(this); when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f}); + when(dictionaryService.isHint(any(), any())).thenReturn(false); } @@ -88,7 +90,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes tempEntity.removeFromGraph(); assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document); + List notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), + document, + TEST_DOSSIER_TEMPLATE_ID); assertTrue(notFoundManualEntities.isEmpty()); assertEquals(1, document.getEntities().size()); } @@ -116,7 +120,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document); + List notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), + document, + TEST_DOSSIER_TEMPLATE_ID); assertEquals(1, notFoundManualEntities.size()); assertTrue(document.getEntities().isEmpty()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index a92223b5..c7ab3c3c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -423,7 +423,7 @@ rule "ETC.0.0: Purity Hint" when $section: Section(containsStringIgnoreCase("purity")) then - entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section) .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end @@ -667,7 +667,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -691,7 +691,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) + $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); @@ -704,7 +704,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: TextEntity(entityType == EntityType.ENTITY, active()) + $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); @@ -713,17 +713,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" // Rule unit: X.6 -rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" +rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); retract($lowerRank); end +rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" + salience 32 + when + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + then + $higherRank.getIntersectingNodes().forEach(node -> update(node)); + $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); + retract($higherRank); + end + //------------------------------------ File attributes rules ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl index 4a190ab0..c4e2088c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl @@ -831,7 +831,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -855,7 +855,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) + $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); @@ -868,7 +868,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: TextEntity(entityType == EntityType.ENTITY, active()) + $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); @@ -880,7 +880,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.7.0: remove all images" salience 512 when - $image: Image(imageType != ImageType.OCR) + $image: Image(imageType != ImageType.OCR, !hasManualChanges()) then $image.remove("X.7.0", "remove all images"); retract($image); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index 7947e2c1..ca0f34c2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -74,8 +74,8 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" $section: Section(containsString("CTL/") || containsString("BL/")) then Stream.concat( - entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section), - entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) + entityCreationService.byString("CTL", "must_redact", EntityType.HINT, $section), + entityCreationService.byString("BL", "must_redact", EntityType.HINT, $section) ).forEach(entity -> entity.skip("SYN.0.0", "hint_only")); end @@ -1012,7 +1012,7 @@ rule "ETC.0.0: Purity Hint" when $section: Section(containsStringIgnoreCase("purity")) then - entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section) .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end @@ -1340,7 +1340,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1364,7 +1364,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) + $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); @@ -1377,7 +1377,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: TextEntity(entityType == EntityType.ENTITY, active()) + $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); @@ -1386,18 +1386,30 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" // Rule unit: X.6 -rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" +rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); retract($lowerRank); end +rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" + salience 32 + when + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + then + $higherRank.getIntersectingNodes().forEach(node -> update(node)); + $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); + retract($higherRank); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index a1e8a1c9..1267cdf4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -91,7 +91,7 @@ rule "H.2.0: Show headlines" when $headline: Headline() then - entityCreationService.bySemanticNode($headline, "headline", EntityType.ENTITY); + entityCreationService.bySemanticNode($headline, "headline", EntityType.HINT); end @@ -1232,7 +1232,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1256,7 +1256,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) + $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); @@ -1269,7 +1269,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: TextEntity(entityType == EntityType.ENTITY, active()) + $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 1f3bdff7..7c8b7beb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -74,8 +74,8 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" $section: Section(containsString("CTL/") || containsString("BL/")) then Stream.concat( - entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section), - entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) + entityCreationService.byString("CTL", "must_redact", EntityType.HINT, $section), + entityCreationService.byString("BL", "must_redact", EntityType.HINT, $section) ).forEach(entity -> entity.skip("SYN.0.0", "hint_only")); end @@ -1051,7 +1051,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1075,7 +1075,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) + $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); @@ -1088,7 +1088,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: TextEntity(entityType == EntityType.ENTITY, active()) + $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); @@ -1097,17 +1097,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" // Rule unit: X.6 -rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" +rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); retract($lowerRank); end +rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" + salience 32 + when + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + then + $higherRank.getIntersectingNodes().forEach(node -> update(node)); + $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); + retract($higherRank); + end + //------------------------------------ File attributes rules ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 198f38cd..ad912d80 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -236,7 +236,7 @@ rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active()) + $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -248,7 +248,7 @@ rule "X.1.0: merge intersecting Entities of same type" salience 64 when $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active()) + $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -264,7 +264,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -277,7 +277,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM salience 64 when $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -288,8 +288,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); @@ -301,8 +301,8 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: TextEntity(entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active()) + $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); @@ -310,17 +310,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" // Rule unit: X.6 -rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" +rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active()) + $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); retract($lowerRank); end +rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" + salience 32 + when + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + then + $higherRank.getIntersectingNodes().forEach(node -> update(node)); + $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); + retract($higherRank); + end + //------------------------------------ File attributes rules ------------------------------------