From ef72cc68612abdb5304c749c26f25565fbbb3690 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 17 Feb 2023 17:06:00 +0100 Subject: [PATCH] RED-6093: Prototype find entities in rules *added improved string to text position mapping --- .../v1/server/redaction/model/Entity.java | 6 +- .../v1/server/redaction/model/Paragraph.java | 61 +-- .../SearchTextWithTextPositionModel.java | 22 + .../redaction/model/SearchableText.java | 31 +- .../v1/server/redaction/model/Section.java | 1 + .../service/DroolsExecutionService.java | 21 +- .../service/EntityRedactionService.java | 34 +- .../SearchTextWithTextPositionFactory.java | 81 ++++ .../redaction/utils/EntitySearchUtils.java | 118 ++++- .../redaction/utils/SearchImplementation.java | 17 +- .../utils/EntitySearchUtilsTest.java | 50 +-- .../src/test/resources/drools/rules.drl | 425 +++--------------- 12 files changed, 353 insertions(+), 514 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index 8931f6ed..53739cc5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -20,7 +20,6 @@ import lombok.NoArgsConstructor; @AllArgsConstructor @EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Entity implements ReasonHolder { - private String word; private String type; private boolean redaction; @@ -30,6 +29,7 @@ public class Entity implements ReasonHolder { private List positionSequences = new ArrayList<>(); private List targetSequences; + @EqualsAndHashCode.Include private Integer start; @EqualsAndHashCode.Include @@ -73,6 +73,7 @@ public class Entity implements ReasonHolder { String headline, int matchedRule, int sectionNumber, + int paragraphNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, @@ -92,6 +93,7 @@ public class Entity implements ReasonHolder { this.headline = headline; this.matchedRule = matchedRule; this.sectionNumber = sectionNumber; + this.paragraphNumber = paragraphNumber; this.legalBasis = legalBasis; this.isDictionaryEntry = isDictionaryEntry; this.textBefore = textBefore; @@ -111,6 +113,7 @@ public class Entity implements ReasonHolder { Integer end, String headline, int sectionNumber, + int paragraphNumber, boolean isDictionaryEntry, boolean isDossierDictionaryEntry, Engine engine, @@ -122,6 +125,7 @@ public class Entity implements ReasonHolder { this.end = end; this.headline = headline; this.sectionNumber = sectionNumber; + this.paragraphNumber = paragraphNumber; this.isDictionaryEntry = isDictionaryEntry; this.isDossierDictionaryEntry = isDossierDictionaryEntry; this.engines.add(engine); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java index a3fb2d2c..8be92642 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java @@ -1,61 +1,16 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; -import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString; - -import java.util.List; - -import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; -import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; - +import lombok.AccessLevel; +import lombok.Builder; import lombok.Getter; +import lombok.experimental.FieldDefaults; @Getter +@Builder +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class Paragraph { - private final String searchText; - private final List textPositions; - private final int[] searchTextToPositions; - private final int sectionNumber; - - private final int start; - private final int end; - - - public Paragraph(TextBlock textBlock, int sectionNumber, int start, int end) { - - textPositions = textBlock.getSequences().stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList(); - this.searchTextToPositions = new int[textPositions.size()]; - this.searchText = buildString(textBlock.getSequences()); - this.start = start; - this.end = end; - this.sectionNumber = sectionNumber; - } - - public boolean containsEntity(Entity entity) { - return entity.getStart() >= start && entity.getEnd() <= end; - } - - private String buildSearchText() { - - StringBuilder searchTextBuilder = new StringBuilder(); - int numberOfNormalCharacters = 0; - - for (int i = 0; i < textPositions.size(); ++i) { - if (isNormalText(textPositions.get(i).getUnicode())) { - searchTextBuilder.append(textPositions.get(i).getUnicode()); - this.searchTextToPositions[numberOfNormalCharacters] = i; - ++numberOfNormalCharacters; - } - } - return searchTextBuilder.toString(); - } - - - private boolean isNormalText(String unicode) { - - return !unicode.equals("\n") && // - !unicode.equals(""); - - } + SearchTextWithTextPositionModel searchTextToTextPosition; + int sectionNumber; + int paragraphNumber; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java new file mode 100644 index 00000000..c849da04 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java @@ -0,0 +1,22 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import java.util.List; +import java.util.Map; + +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.experimental.FieldDefaults; + +@Builder +@Getter +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class SearchTextWithTextPositionModel { + + String searchText; + Map stringCoordsToTextPositionCoords; + List lineBreaksStringCoords; + List textPositionSequences; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java index 8e45908c..0e12be13 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java @@ -1,5 +1,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; @@ -7,12 +13,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - import lombok.Getter; public class SearchableText { @@ -186,30 +186,13 @@ public class SearchableText { return stringRepresentation; } - -// public List getParagraphStrings(){ -// -// } - - public static String buildString(List sequences) { StringBuilder sb = new StringBuilder(); - - TextPositionSequence previous = null; for (TextPositionSequence word : sequences) { - - if (previous != null) { - if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) { - sb.append('\n'); - } else { - sb.append(' '); - } - } sb.append(word.toString()); - previous = word; + sb.append(' '); } - return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " "); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index c0e92faa..e5ef750c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1372,6 +1372,7 @@ public class Section { value.getRowSpanStart() + word.length(), headline, sectionNumber, + -1, false, false, Engine.RULE, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 3a21f8ac..eb1f88e7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -4,10 +4,9 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.kie.api.KieServices; @@ -16,6 +15,8 @@ import org.kie.api.builder.KieFileSystem; import org.kie.api.builder.KieModule; import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieSession; +import org.kie.api.runtime.rule.QueryResults; +import org.kie.api.runtime.rule.QueryResultsRow; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.server.client.RulesClient; @@ -51,19 +52,29 @@ public class DroolsExecutionService { @Timed("redactmanager_executeRules") - public List
executeRules(KieContainer kieContainer, List
sections, List paragraphs, Dictionary dictionary) { + public List executeRules(KieContainer kieContainer, List
sections, List paragraphs, Dictionary dictionary) { - Set entities = new HashSet<>(); KieSession kieSession = kieContainer.newKieSession(); + kieSession.setGlobal("dictionary", dictionary); sections.forEach(kieSession::insert); paragraphs.forEach(kieSession::insert); kieSession.fireAllRules(); + List entities = getEntities(kieSession); kieSession.dispose(); - return sections; + return entities; } + public List getEntities(KieSession ks) { + List entities = new LinkedList<>(); + QueryResults entitiesResult = ks.getQueryResults("getEntities"); + for (QueryResultsRow resultsRow : entitiesResult) { + entities.add((Entity) resultsRow.get("$result")); + } + return entities; + } + public KieContainer updateRules(String dossierTemplateId) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index cc0d1461..f6a0f771 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,7 +1,5 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString; - import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -47,6 +45,7 @@ public class EntityRedactionService { private final RedactionServiceSettings redactionServiceSettings; private final DroolsExecutionService droolsExecutionService; + private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory; public PageEntities findEntities(Dictionary dictionary, List sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) { @@ -126,7 +125,6 @@ public class EntityRedactionService { } Set addedFileAttributes = new HashSet<>(); - Set entities = new HashSet<>(); sections.forEach(section -> { if (!addedFileAttributes.isEmpty()) { @@ -138,28 +136,27 @@ public class EntityRedactionService { } }); - List paragraphs = new ArrayList<>(); for (var reanalysisSection : reanalysisSections) { - int start = 0; - int end; - for (var textBlock : reanalysisSection.getTextBlocks()) { - int length = buildString(textBlock.getSequences()).length(); - end = start + length; - paragraphs.add(new Paragraph(textBlock, reanalysisSection.getSectionNumber(), start, end)); - start = end + 1; + for (int i = 0; i < reanalysisSection.getTextBlocks().size(); ++i) { + paragraphs.add(Paragraph.builder() + .paragraphNumber(i) + .sectionNumber(reanalysisSection.getSectionNumber()) + .searchTextToTextPosition(searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(reanalysisSection.getTextBlocks().get(i).getSequences())) + .build()); } } - List
analysedSections = droolsExecutionService.executeRules(kieContainer, sections, paragraphs, dictionary); + List entitiesList = droolsExecutionService.executeRules(kieContainer, sections, paragraphs, dictionary); + Set entities = new HashSet<>(entitiesList); - analysedSections.forEach(analysedSection -> { + sections.forEach(analysedSection -> { addedFileAttributes.addAll(analysedSection.getAddedFileAttributes()); - EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities()); + EntitySearchUtils.removeEntitiesContainedInLarger(entities); - var entriesWithoutSurroundingText = analysedSection.getEntities() - .stream() + var entriesWithoutSurroundingText = entities.stream() + .filter(e -> e.getSectionNumber() == analysedSection.getSectionNumber()) .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null) .collect(Collectors.toSet()); @@ -178,8 +175,6 @@ public class EntityRedactionService { redactionServiceSettings.getNumberOfSurroundingWords()); } - entities.addAll(analysedSection.getEntities()); - if (!local) { for (Image image : analysedSection.getImages()) { imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image); @@ -212,6 +207,7 @@ public class EntityRedactionService { entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), + -1, entity.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), @@ -315,6 +311,7 @@ public class EntityRedactionService { res.getEndOffset(), headline, sectionNumber, + -1, false, false, Engine.NER, @@ -334,6 +331,7 @@ public class EntityRedactionService { res.getEndOffset(), headline, sectionNumber, + -1, false, false, Engine.NER, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java new file mode 100644 index 00000000..a059a63a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java @@ -0,0 +1,81 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel; + +@Service +public class SearchTextWithTextPositionFactory { + + public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List sequences) { + + Map stringIdxToPositionIdx = new HashMap<>(); + List lineBreaksStringIdx = new LinkedList<>(); + StringBuilder sb = new StringBuilder(); + + int stringIdx = 0; + int positionIdx = 0; + + String currentUnicode; + String previousUnicode = " "; + + for (TextPositionSequence word : sequences) { + for (int i = 0; i < word.getTextPositions().size(); ++i) { + + currentUnicode = word.getTextPositions().get(i).getUnicode(); + + if (isLineBreak(currentUnicode)) { + lineBreaksStringIdx.add(stringIdx); + } else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && // + !isHyphenLinebreak(currentUnicode)) { + + sb.append(currentUnicode); + stringIdxToPositionIdx.put(stringIdx, positionIdx); + ++stringIdx; + } + + previousUnicode = currentUnicode; + ++positionIdx; + } + + previousUnicode = " "; + sb.append(previousUnicode); + stringIdxToPositionIdx.put(stringIdx, positionIdx); + ++stringIdx; + } + + return SearchTextWithTextPositionModel.builder() + .searchText(sb.toString().stripTrailing()) + .lineBreaksStringCoords(lineBreaksStringIdx) + .stringCoordsToTextPositionCoords(stringIdxToPositionIdx) + .textPositionSequences(sequences) + .build(); + } + + + private boolean isLineBreak(String currentUnicode) { + + return Objects.equals(currentUnicode, "\n"); + } + + + private boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) { + + return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " "); + } + + + private boolean isHyphenLinebreak(String unicodeCharacter) { + + return unicodeCharacter.matches("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R"); + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 8203983e..1f3cc89c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -5,20 +5,25 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import lombok.experimental.UtilityClass; @@ -29,6 +34,109 @@ import lombok.extern.slf4j.Slf4j; @SuppressWarnings("PMD") public final class EntitySearchUtils { + public static List findEntityPositionSequences(String entityString, SearchTextWithTextPositionModel searchTextWithTextPositionModel) { + + return findOccurrences(entityString, searchTextWithTextPositionModel.getSearchText()).stream() + .map(range -> mapToTextPositionCoordinates(range, searchTextWithTextPositionModel.getStringCoordsToTextPositionCoords())) + .map(range -> mapToTextPositions(range, searchTextWithTextPositionModel.getTextPositionSequences())) + .map(textPositionSequences -> mapToEntityPositionSequence(entityString, textPositionSequences)) + .toList(); + } + + + private static List mapToTextPositions(IndexRange range, List sequences) { + + List sequencesInRange = new LinkedList<>(); + int sequenceStartIdx = 0; + for (var sequence : sequences) { + if (sequenceBeforeRange(sequenceStartIdx + sequence.length(), range)) { + sequenceStartIdx = sequenceStartIdx + sequence.length(); + } else if (sequenceBehindRange(sequenceStartIdx, range)) { + return sequencesInRange; + } else if (fullSequenceInRange(sequenceStartIdx, sequence.length(), range)) { + sequencesInRange.add(sequence); + sequenceStartIdx = sequenceStartIdx + sequence.length(); + } else if (partialSequenceInRange(sequenceStartIdx, sequence.length(), range)) { + sequencesInRange.add(sliceTextPositionSequence(sequence, range.startIdx - sequenceStartIdx, range.endIdx - sequenceStartIdx)); + sequenceStartIdx = sequenceStartIdx + sequence.length(); + } + } + return sequencesInRange; + } + + + private static EntityPositionSequence mapToEntityPositionSequence(String entityString, List textPositionSequences) { + + return EntityPositionSequence.builder() // + .id(entityString) // + .pageNumber(textPositionSequences.get(0).getPage()) // + .sequences(textPositionSequences) // + .build(); + } + + + private static TextPositionSequence sliceTextPositionSequence(TextPositionSequence sequence, int start, int end) { + + int startIndex = Math.max(start, 0); + int endIndex = Math.min(end, sequence.length()); + return TextPositionSequence.builder() + .dir(sequence.getDir()) + .page(sequence.getPage()) + .pageHeight(sequence.getPageHeight()) + .pageWidth(sequence.getPageWidth()) + .rotation(sequence.getRotation()) + .textPositions(sequence.getTextPositions().subList(startIndex, endIndex)) + .build(); + } + + + private static boolean sequenceBeforeRange(int end, IndexRange range) { + + return end <= range.startIdx; + } + + + private static boolean sequenceBehindRange(int start, IndexRange range) { + + return start >= range.endIdx; + } + + + private static boolean partialSequenceInRange(int start, int length, IndexRange range) { + + return start >= range.startIdx || (start + length) <= range.endIdx; + } + + + private static boolean fullSequenceInRange(int start, int length, IndexRange range) { + + return start >= range.startIdx && (start + length) <= range.endIdx; + } + + + private static IndexRange mapToTextPositionCoordinates(IndexRange range, Map stringCoordsToTextPositionCoords) { + + return new IndexRange(stringCoordsToTextPositionCoords.get(range.startIdx), stringCoordsToTextPositionCoords.get(range.endIdx)); + } + + + private static List findOccurrences(String subString, String searchText) { + + int startIndex = 0; + List occurrenceRanges = new LinkedList<>(); + + Pattern pattern = Pattern.compile(subString); + Matcher matcher = pattern.matcher(searchText); + + while (matcher.find(startIndex)) { + occurrenceRanges.add(new IndexRange(matcher.start(), matcher.end())); + startIndex = matcher.end() + 1; + } + + return occurrenceRanges; + } + + public boolean sectionContainsAny(String sectionText, SearchImplementation searchImplementation) { return searchImplementation.atLeastOneMatches(sectionText); @@ -63,7 +171,7 @@ public final class EntitySearchUtils { Set entities = new HashSet<>(); - searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.getStartIndex(), match.getEndIndex())); + searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.startIndex(), match.endIndex())); return entities; } @@ -79,6 +187,7 @@ public final class EntitySearchUtils { stopIndex, findEntityDetails.getHeadline(), findEntityDetails.getSectionNumber(), + -1, findEntityDetails.isDictionaryEntry(), findEntityDetails.isDossierDictionary(), findEntityDetails.getEngine(), @@ -358,7 +467,7 @@ public final class EntitySearchUtils { for (Entity existing : existingEntities) { - // skip if either start or end is equal + // skip if either startIdx or endIdx is equal if (existing.getStart().equals(found.getStart()) || existing.getEnd().equals(found.getEnd())) { continue; } @@ -374,4 +483,9 @@ public final class EntitySearchUtils { return false; } + + record IndexRange(int startIdx, int endIdx) { + + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java index 4cfb5e6a..c1270672 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; -import lombok.AllArgsConstructor; -import lombok.Data; - -import org.ahocorasick.trie.Trie; - import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -12,6 +7,10 @@ import java.util.Locale; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.ahocorasick.trie.Trie; + +import lombok.Data; + @Data public class SearchImplementation { @@ -94,13 +93,7 @@ public class SearchImplementation { } } - - @Data - @AllArgsConstructor - public static class MatchPosition { - - private int startIndex; - private int endIndex; + public record MatchPosition(int startIndex, int endIndex) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java index f371cd93..708eddfb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java @@ -17,8 +17,8 @@ public class EntitySearchUtilsTest { public void testNestedEntitiesRemoval() { Set entities = new HashSet<>(); - Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); entities.add(nested); entities.add(nesting); EntitySearchUtils.removeEntitiesContainedInLarger(entities); @@ -40,14 +40,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -73,14 +73,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -105,14 +105,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -137,15 +137,15 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); foundEntities.add(foundEntities3); @@ -171,16 +171,16 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntitiesOverlap1); foundEntities.add(foundEntitiesOverlap2); foundEntities.add(foundEntitiesSubset1); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 42b44ab3..8695d99f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1,390 +1,67 @@ package drools -import com.iqser.red.service.redaction.v1.server.redaction.model.* +import static java.lang.String.format; +import java.util.List; +import java.util.LinkedList; +import java.util.HashSet; +import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.model.Engine; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import java.util.Set; + +global Dictionary dictionary + +query "getEntities" + $result: Entity() +end // --------------------------------------- AI rules ------------------------------------------------------------------- -rule "-1: find entities from dictionary" +rule "0: find entities from entry dictionary" salience 100 - no-loop true when - section: Section(!text.isEmpty()) + $section: Section() + $paragraph: Paragraph(sectionNumber == $section.sectionNumber) + $entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition()) then - section.findDictionaryEntities(); - update(section); - end - -rule "0: Add CBI_author from ai" - when - section: Section(aiMatchesType("CBI_author")) - then - section.addAiEntities("CBI_author", "CBI_author"); - end - -rule "0: Combine address parts from ai to CBI_address (org is mandatory)" - when - section: Section(aiMatchesType("ORG")) - then - section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); - end - -rule "0: Combine address parts from ai to CBI_address (street is mandatory)" - when - section: Section(aiMatchesType("STREET")) - then - section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); - end - -rule "0: Combine address parts from ai to CBI_address (city is mandatory)" - when - section: Section(aiMatchesType("CITY")) - then - section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false); - end - - -// --------------------------------------- CBI rules ------------------------------------------------------------------- - -rule "1: Redact CBI Authors (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) - then - section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "2: Redact CBI Authors (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) - then - section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "3: Redact not CBI Address (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) - then - section.redactNot("CBI_address", 3, "Address found for non vertebrate study"); - section.ignoreRecommendations("CBI_address"); - end - -rule "4: Redact CBI Address (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) - then - section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "5: Do not redact genitive CBI_author" - when - section: Section(matchesType("CBI_author")) - then - section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); - end - - -rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) - then - section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) - then - section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "14: Redact and add recommendation for et al. author (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "15: Redact and add recommendation for et al. author (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "16: Add recommendation for Addresses in Test Organism sections" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:")) - then - section.recommendLineAfter("Source:", "CBI_address"); - end - -rule "17: Add recommendation for Addresses in Test Animals sections" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source")) - then - section.recommendLineAfter("Source", "CBI_address"); + List eps = new LinkedList<>(); + eps.add($entityPositionSequence); + Set engineSet = new HashSet<>(); + engineSet.add(Engine.DICTIONARY); + Set empty = new HashSet<>(); + insert(new Entity("CBI_author", + "custom", + true, + format("Found in Dictionary: %s", "custom"), + eps, + $section.getHeadline(), + 0, + $section.getSectionNumber(), + $paragraph.getParagraphNumber(), + "", + true, + "", + "", + -1, + -1, + false, + engineSet, + empty, + EntityType.ENTITY + )); end /* - -rule "18: Do not redact Names and Addresses if Published Information found" - when - section: Section(matchesType("published_information")) - then - section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found"); - section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found"); - end - -*/ - -// --------------------------------------- PII rules ------------------------------------------------------------------- - - -rule "19: Redacted PII Personal Identification Information (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) - then - section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "20: Redacted PII Personal Identification Information (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) - then - section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "21: Redact Emails by RegEx (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) - then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "22: Redact Emails by RegEx (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) - then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "23: Redact contact information (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") - || text.contains("Contact:") - || text.contains("Alternative contact:") - || (text.contains("No:") && text.contains("Fax")) - || (text.contains("Contact:") && text.contains("Tel.:")) - || text.contains("European contact:") - )) - then - section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "24: Redact contact information (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") - || text.contains("Contact:") - || text.contains("Alternative contact:") - || (text.contains("No:") && text.contains("Fax")) - || (text.contains("Contact:") && text.contains("Tel.:")) - || text.contains("European contact:") - )) - then - section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( - text.contains("Contact") - || text.contains("Telephone") - || text.contains("Phone") - || text.contains("Fax") - || text.contains("Tel") - || text.contains("Ter") - || text.contains("Mobile") - || text.contains("Fel") - || text.contains("Fer") - )) - then - section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( - text.contains("Contact") - || text.contains("Telephone") - || text.contains("Phone") - || text.contains("Fax") - || text.contains("Tel") - || text.contains("Ter") - || text.contains("Mobile") - || text.contains("Fel") - || text.contains("Fer") - )) - then - section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "27: Redact AUTHOR(S) (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("COMPLETION DATE:") - && !searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "28: Redact AUTHOR(S) (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("COMPLETION DATE:") - && !searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "29: Redact AUTHOR(S) (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "30: Redact AUTHOR(S) (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("PERFORMING LABORATORY:") - ) - then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); - end - -rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("PERFORMING LABORATORY:")) - then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "33: Redact study director abbreviation" - when - section: Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML"))) - then - section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - - -// --------------------------------------- other rules ------------------------------------------------------------------- - -rule "34: Purity Hint" - when - section: Section(searchText.toLowerCase().contains("purity")) - then - section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); - end - - - -rule "35: Redact signatures (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) - then - section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "36: Redact signatures (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) - then - section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "43: Redact Logos (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo")) - then - section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "44: Redact Author if contained in paragraph with 'unpublished'" +rule "44: Don't redact Author if contained in paragraph with published information" when $section: Section() - $entity: Entity(type == "CBI_author") from $section.getEntities() - $publishedEntity: Entity(type == "published_information") from $section.getEntities() - $paragraph: Paragraph(sectionNumber == $section.sectionNumber) - then - if ($paragraph.containsEntity($entity) && $paragraph.containsEntity($publishedEntity) ) { + $paragraph: Paragraph($section.sectionNumber == sectionNumber) - $entity.setRedaction(false); - } + $authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber()) + $publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber()) + then + $authorEntity.setRedaction(false); end +*/ \ No newline at end of file