RED-6093: Prototype find entities in rules
*added improved string to text position mapping
This commit is contained in:
Kilian Schuettler
parent
66b2d52d40
commit
ef72cc6861
@ -20,7 +20,6 @@ import lombok.NoArgsConstructor;
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Entity implements ReasonHolder {
|
||||
|
||||
private String word;
|
||||
private String type;
|
||||
private boolean redaction;
|
||||
@ -30,6 +29,7 @@ public class Entity implements ReasonHolder {
|
||||
private List<EntityPositionSequence> positionSequences = new ArrayList<>();
|
||||
private List<TextPositionSequence> targetSequences;
|
||||
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer start;
|
||||
@EqualsAndHashCode.Include
|
||||
@ -73,6 +73,7 @@ public class Entity implements ReasonHolder {
|
||||
String headline,
|
||||
int matchedRule,
|
||||
int sectionNumber,
|
||||
int paragraphNumber,
|
||||
String legalBasis,
|
||||
boolean isDictionaryEntry,
|
||||
String textBefore,
|
||||
@ -92,6 +93,7 @@ public class Entity implements ReasonHolder {
|
||||
this.headline = headline;
|
||||
this.matchedRule = matchedRule;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.paragraphNumber = paragraphNumber;
|
||||
this.legalBasis = legalBasis;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.textBefore = textBefore;
|
||||
@ -111,6 +113,7 @@ public class Entity implements ReasonHolder {
|
||||
Integer end,
|
||||
String headline,
|
||||
int sectionNumber,
|
||||
int paragraphNumber,
|
||||
boolean isDictionaryEntry,
|
||||
boolean isDossierDictionaryEntry,
|
||||
Engine engine,
|
||||
@ -122,6 +125,7 @@ public class Entity implements ReasonHolder {
|
||||
this.end = end;
|
||||
this.headline = headline;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.paragraphNumber = paragraphNumber;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.isDossierDictionaryEntry = isDossierDictionaryEntry;
|
||||
this.engines.add(engine);
|
||||
|
||||
@ -1,61 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class Paragraph {
|
||||
|
||||
private final String searchText;
|
||||
private final List<RedTextPosition> textPositions;
|
||||
private final int[] searchTextToPositions;
|
||||
private final int sectionNumber;
|
||||
|
||||
private final int start;
|
||||
private final int end;
|
||||
|
||||
|
||||
public Paragraph(TextBlock textBlock, int sectionNumber, int start, int end) {
|
||||
|
||||
textPositions = textBlock.getSequences().stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
|
||||
this.searchTextToPositions = new int[textPositions.size()];
|
||||
this.searchText = buildString(textBlock.getSequences());
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.sectionNumber = sectionNumber;
|
||||
}
|
||||
|
||||
public boolean containsEntity(Entity entity) {
|
||||
return entity.getStart() >= start && entity.getEnd() <= end;
|
||||
}
|
||||
|
||||
private String buildSearchText() {
|
||||
|
||||
StringBuilder searchTextBuilder = new StringBuilder();
|
||||
int numberOfNormalCharacters = 0;
|
||||
|
||||
for (int i = 0; i < textPositions.size(); ++i) {
|
||||
if (isNormalText(textPositions.get(i).getUnicode())) {
|
||||
searchTextBuilder.append(textPositions.get(i).getUnicode());
|
||||
this.searchTextToPositions[numberOfNormalCharacters] = i;
|
||||
++numberOfNormalCharacters;
|
||||
}
|
||||
}
|
||||
return searchTextBuilder.toString();
|
||||
}
|
||||
|
||||
|
||||
private boolean isNormalText(String unicode) {
|
||||
|
||||
return !unicode.equals("\n") && //
|
||||
!unicode.equals("");
|
||||
|
||||
}
|
||||
SearchTextWithTextPositionModel searchTextToTextPosition;
|
||||
int sectionNumber;
|
||||
int paragraphNumber;
|
||||
}
|
||||
|
||||
@ -0,0 +1,22 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Builder
|
||||
@Getter
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class SearchTextWithTextPositionModel {
|
||||
|
||||
String searchText;
|
||||
Map<Integer, Integer> stringCoordsToTextPositionCoords;
|
||||
List<Integer> lineBreaksStringCoords;
|
||||
List<TextPositionSequence> textPositionSequences;
|
||||
}
|
||||
@ -1,5 +1,11 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
@ -7,12 +13,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
public class SearchableText {
|
||||
@ -186,30 +186,13 @@ public class SearchableText {
|
||||
return stringRepresentation;
|
||||
}
|
||||
|
||||
|
||||
// public List<String> getParagraphStrings(){
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
public static String buildString(List<TextPositionSequence> sequences) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : sequences) {
|
||||
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
sb.append(' ');
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
|
||||
}
|
||||
|
||||
|
||||
@ -1372,6 +1372,7 @@ public class Section {
|
||||
value.getRowSpanStart() + word.length(),
|
||||
headline,
|
||||
sectionNumber,
|
||||
-1,
|
||||
false,
|
||||
false,
|
||||
Engine.RULE,
|
||||
|
||||
@ -4,10 +4,9 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.KieServices;
|
||||
@ -16,6 +15,8 @@ import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.kie.api.runtime.KieSession;
|
||||
import org.kie.api.runtime.rule.QueryResults;
|
||||
import org.kie.api.runtime.rule.QueryResultsRow;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
@ -51,19 +52,29 @@ public class DroolsExecutionService {
|
||||
|
||||
|
||||
@Timed("redactmanager_executeRules")
|
||||
public List<Section> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
|
||||
public List<Entity> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
KieSession kieSession = kieContainer.newKieSession();
|
||||
kieSession.setGlobal("dictionary", dictionary);
|
||||
sections.forEach(kieSession::insert);
|
||||
paragraphs.forEach(kieSession::insert);
|
||||
kieSession.fireAllRules();
|
||||
List<Entity> entities = getEntities(kieSession);
|
||||
kieSession.dispose();
|
||||
|
||||
return sections;
|
||||
return entities;
|
||||
|
||||
}
|
||||
|
||||
public List<Entity> getEntities(KieSession ks) {
|
||||
List<Entity> entities = new LinkedList<>();
|
||||
QueryResults entitiesResult = ks.getQueryResults("getEntities");
|
||||
for (QueryResultsRow resultsRow : entitiesResult) {
|
||||
entities.add((Entity) resultsRow.get("$result"));
|
||||
}
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
public KieContainer updateRules(String dossierTemplateId) {
|
||||
|
||||
|
||||
@ -1,7 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@ -47,6 +45,7 @@ public class EntityRedactionService {
|
||||
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory;
|
||||
|
||||
|
||||
public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
|
||||
@ -126,7 +125,6 @@ public class EntityRedactionService {
|
||||
}
|
||||
|
||||
Set<FileAttribute> addedFileAttributes = new HashSet<>();
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
sections.forEach(section -> {
|
||||
|
||||
if (!addedFileAttributes.isEmpty()) {
|
||||
@ -138,28 +136,27 @@ public class EntityRedactionService {
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
List<Paragraph> paragraphs = new ArrayList<>();
|
||||
for (var reanalysisSection : reanalysisSections) {
|
||||
int start = 0;
|
||||
int end;
|
||||
for (var textBlock : reanalysisSection.getTextBlocks()) {
|
||||
int length = buildString(textBlock.getSequences()).length();
|
||||
end = start + length;
|
||||
paragraphs.add(new Paragraph(textBlock, reanalysisSection.getSectionNumber(), start, end));
|
||||
start = end + 1;
|
||||
for (int i = 0; i < reanalysisSection.getTextBlocks().size(); ++i) {
|
||||
paragraphs.add(Paragraph.builder()
|
||||
.paragraphNumber(i)
|
||||
.sectionNumber(reanalysisSection.getSectionNumber())
|
||||
.searchTextToTextPosition(searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(reanalysisSection.getTextBlocks().get(i).getSequences()))
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
List<Section> analysedSections = droolsExecutionService.executeRules(kieContainer, sections, paragraphs, dictionary);
|
||||
List<Entity> entitiesList = droolsExecutionService.executeRules(kieContainer, sections, paragraphs, dictionary);
|
||||
Set<Entity> entities = new HashSet<>(entitiesList);
|
||||
|
||||
analysedSections.forEach(analysedSection -> {
|
||||
sections.forEach(analysedSection -> {
|
||||
addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
|
||||
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
|
||||
var entriesWithoutSurroundingText = analysedSection.getEntities()
|
||||
.stream()
|
||||
var entriesWithoutSurroundingText = entities.stream()
|
||||
.filter(e -> e.getSectionNumber() == analysedSection.getSectionNumber())
|
||||
.filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
@ -178,8 +175,6 @@ public class EntityRedactionService {
|
||||
redactionServiceSettings.getNumberOfSurroundingWords());
|
||||
}
|
||||
|
||||
entities.addAll(analysedSection.getEntities());
|
||||
|
||||
if (!local) {
|
||||
for (Image image : analysedSection.getImages()) {
|
||||
imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
|
||||
@ -212,6 +207,7 @@ public class EntityRedactionService {
|
||||
entity.getHeadline(),
|
||||
entity.getMatchedRule(),
|
||||
entity.getSectionNumber(),
|
||||
-1,
|
||||
entity.getLegalBasis(),
|
||||
entity.isDictionaryEntry(),
|
||||
entity.getTextBefore(),
|
||||
@ -315,6 +311,7 @@ public class EntityRedactionService {
|
||||
res.getEndOffset(),
|
||||
headline,
|
||||
sectionNumber,
|
||||
-1,
|
||||
false,
|
||||
false,
|
||||
Engine.NER,
|
||||
@ -334,6 +331,7 @@ public class EntityRedactionService {
|
||||
res.getEndOffset(),
|
||||
headline,
|
||||
sectionNumber,
|
||||
-1,
|
||||
false,
|
||||
false,
|
||||
Engine.NER,
|
||||
|
||||
@ -0,0 +1,81 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
|
||||
|
||||
@Service
|
||||
public class SearchTextWithTextPositionFactory {
|
||||
|
||||
public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
|
||||
|
||||
Map<Integer, Integer> stringIdxToPositionIdx = new HashMap<>();
|
||||
List<Integer> lineBreaksStringIdx = new LinkedList<>();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
int stringIdx = 0;
|
||||
int positionIdx = 0;
|
||||
|
||||
String currentUnicode;
|
||||
String previousUnicode = " ";
|
||||
|
||||
for (TextPositionSequence word : sequences) {
|
||||
for (int i = 0; i < word.getTextPositions().size(); ++i) {
|
||||
|
||||
currentUnicode = word.getTextPositions().get(i).getUnicode();
|
||||
|
||||
if (isLineBreak(currentUnicode)) {
|
||||
lineBreaksStringIdx.add(stringIdx);
|
||||
} else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && //
|
||||
!isHyphenLinebreak(currentUnicode)) {
|
||||
|
||||
sb.append(currentUnicode);
|
||||
stringIdxToPositionIdx.put(stringIdx, positionIdx);
|
||||
++stringIdx;
|
||||
}
|
||||
|
||||
previousUnicode = currentUnicode;
|
||||
++positionIdx;
|
||||
}
|
||||
|
||||
previousUnicode = " ";
|
||||
sb.append(previousUnicode);
|
||||
stringIdxToPositionIdx.put(stringIdx, positionIdx);
|
||||
++stringIdx;
|
||||
}
|
||||
|
||||
return SearchTextWithTextPositionModel.builder()
|
||||
.searchText(sb.toString().stripTrailing())
|
||||
.lineBreaksStringCoords(lineBreaksStringIdx)
|
||||
.stringCoordsToTextPositionCoords(stringIdxToPositionIdx)
|
||||
.textPositionSequences(sequences)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private boolean isLineBreak(String currentUnicode) {
|
||||
|
||||
return Objects.equals(currentUnicode, "\n");
|
||||
}
|
||||
|
||||
|
||||
private boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) {
|
||||
|
||||
return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " ");
|
||||
}
|
||||
|
||||
|
||||
private boolean isHyphenLinebreak(String unicodeCharacter) {
|
||||
|
||||
return unicodeCharacter.matches("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -5,20 +5,25 @@ import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
@ -29,6 +34,109 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@SuppressWarnings("PMD")
|
||||
public final class EntitySearchUtils {
|
||||
|
||||
public static List<EntityPositionSequence> findEntityPositionSequences(String entityString, SearchTextWithTextPositionModel searchTextWithTextPositionModel) {
|
||||
|
||||
return findOccurrences(entityString, searchTextWithTextPositionModel.getSearchText()).stream()
|
||||
.map(range -> mapToTextPositionCoordinates(range, searchTextWithTextPositionModel.getStringCoordsToTextPositionCoords()))
|
||||
.map(range -> mapToTextPositions(range, searchTextWithTextPositionModel.getTextPositionSequences()))
|
||||
.map(textPositionSequences -> mapToEntityPositionSequence(entityString, textPositionSequences))
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
private static List<TextPositionSequence> mapToTextPositions(IndexRange range, List<TextPositionSequence> sequences) {
|
||||
|
||||
List<TextPositionSequence> sequencesInRange = new LinkedList<>();
|
||||
int sequenceStartIdx = 0;
|
||||
for (var sequence : sequences) {
|
||||
if (sequenceBeforeRange(sequenceStartIdx + sequence.length(), range)) {
|
||||
sequenceStartIdx = sequenceStartIdx + sequence.length();
|
||||
} else if (sequenceBehindRange(sequenceStartIdx, range)) {
|
||||
return sequencesInRange;
|
||||
} else if (fullSequenceInRange(sequenceStartIdx, sequence.length(), range)) {
|
||||
sequencesInRange.add(sequence);
|
||||
sequenceStartIdx = sequenceStartIdx + sequence.length();
|
||||
} else if (partialSequenceInRange(sequenceStartIdx, sequence.length(), range)) {
|
||||
sequencesInRange.add(sliceTextPositionSequence(sequence, range.startIdx - sequenceStartIdx, range.endIdx - sequenceStartIdx));
|
||||
sequenceStartIdx = sequenceStartIdx + sequence.length();
|
||||
}
|
||||
}
|
||||
return sequencesInRange;
|
||||
}
|
||||
|
||||
|
||||
private static EntityPositionSequence mapToEntityPositionSequence(String entityString, List<TextPositionSequence> textPositionSequences) {
|
||||
|
||||
return EntityPositionSequence.builder() //
|
||||
.id(entityString) //
|
||||
.pageNumber(textPositionSequences.get(0).getPage()) //
|
||||
.sequences(textPositionSequences) //
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private static TextPositionSequence sliceTextPositionSequence(TextPositionSequence sequence, int start, int end) {
|
||||
|
||||
int startIndex = Math.max(start, 0);
|
||||
int endIndex = Math.min(end, sequence.length());
|
||||
return TextPositionSequence.builder()
|
||||
.dir(sequence.getDir())
|
||||
.page(sequence.getPage())
|
||||
.pageHeight(sequence.getPageHeight())
|
||||
.pageWidth(sequence.getPageWidth())
|
||||
.rotation(sequence.getRotation())
|
||||
.textPositions(sequence.getTextPositions().subList(startIndex, endIndex))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private static boolean sequenceBeforeRange(int end, IndexRange range) {
|
||||
|
||||
return end <= range.startIdx;
|
||||
}
|
||||
|
||||
|
||||
private static boolean sequenceBehindRange(int start, IndexRange range) {
|
||||
|
||||
return start >= range.endIdx;
|
||||
}
|
||||
|
||||
|
||||
private static boolean partialSequenceInRange(int start, int length, IndexRange range) {
|
||||
|
||||
return start >= range.startIdx || (start + length) <= range.endIdx;
|
||||
}
|
||||
|
||||
|
||||
private static boolean fullSequenceInRange(int start, int length, IndexRange range) {
|
||||
|
||||
return start >= range.startIdx && (start + length) <= range.endIdx;
|
||||
}
|
||||
|
||||
|
||||
private static IndexRange mapToTextPositionCoordinates(IndexRange range, Map<Integer, Integer> stringCoordsToTextPositionCoords) {
|
||||
|
||||
return new IndexRange(stringCoordsToTextPositionCoords.get(range.startIdx), stringCoordsToTextPositionCoords.get(range.endIdx));
|
||||
}
|
||||
|
||||
|
||||
private static List<IndexRange> findOccurrences(String subString, String searchText) {
|
||||
|
||||
int startIndex = 0;
|
||||
List<IndexRange> occurrenceRanges = new LinkedList<>();
|
||||
|
||||
Pattern pattern = Pattern.compile(subString);
|
||||
Matcher matcher = pattern.matcher(searchText);
|
||||
|
||||
while (matcher.find(startIndex)) {
|
||||
occurrenceRanges.add(new IndexRange(matcher.start(), matcher.end()));
|
||||
startIndex = matcher.end() + 1;
|
||||
}
|
||||
|
||||
return occurrenceRanges;
|
||||
}
|
||||
|
||||
|
||||
public boolean sectionContainsAny(String sectionText, SearchImplementation searchImplementation) {
|
||||
|
||||
return searchImplementation.atLeastOneMatches(sectionText);
|
||||
@ -63,7 +171,7 @@ public final class EntitySearchUtils {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
|
||||
searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.getStartIndex(), match.getEndIndex()));
|
||||
searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.startIndex(), match.endIndex()));
|
||||
|
||||
return entities;
|
||||
}
|
||||
@ -79,6 +187,7 @@ public final class EntitySearchUtils {
|
||||
stopIndex,
|
||||
findEntityDetails.getHeadline(),
|
||||
findEntityDetails.getSectionNumber(),
|
||||
-1,
|
||||
findEntityDetails.isDictionaryEntry(),
|
||||
findEntityDetails.isDossierDictionary(),
|
||||
findEntityDetails.getEngine(),
|
||||
@ -358,7 +467,7 @@ public final class EntitySearchUtils {
|
||||
|
||||
for (Entity existing : existingEntities) {
|
||||
|
||||
// skip if either start or end is equal
|
||||
// skip if either startIdx or endIdx is equal
|
||||
if (existing.getStart().equals(found.getStart()) || existing.getEnd().equals(found.getEnd())) {
|
||||
continue;
|
||||
}
|
||||
@ -374,4 +483,9 @@ public final class EntitySearchUtils {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
record IndexRange(int startIdx, int endIdx) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,10 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import org.ahocorasick.trie.Trie;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
@ -12,6 +7,10 @@ import java.util.Locale;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.ahocorasick.trie.Trie;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class SearchImplementation {
|
||||
|
||||
@ -94,13 +93,7 @@ public class SearchImplementation {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public static class MatchPosition {
|
||||
|
||||
private int startIndex;
|
||||
private int endIndex;
|
||||
public record MatchPosition(int startIndex, int endIndex) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -17,8 +17,8 @@ public class EntitySearchUtilsTest {
|
||||
public void testNestedEntitiesRemoval() {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
entities.add(nested);
|
||||
entities.add(nesting);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
@ -40,14 +40,14 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntities1);
|
||||
foundEntities.add(foundEntities2);
|
||||
|
||||
@ -73,14 +73,14 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntities1);
|
||||
foundEntities.add(foundEntities2);
|
||||
|
||||
@ -105,14 +105,14 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntities1);
|
||||
foundEntities.add(foundEntities2);
|
||||
|
||||
@ -137,15 +137,15 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntities1);
|
||||
foundEntities.add(foundEntities2);
|
||||
foundEntities.add(foundEntities3);
|
||||
@ -171,16 +171,16 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntitiesOverlap1);
|
||||
foundEntities.add(foundEntitiesOverlap2);
|
||||
foundEntities.add(foundEntitiesSubset1);
|
||||
|
||||
@ -1,390 +1,67 @@
|
||||
package drools
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.LinkedList;
|
||||
import java.util.HashSet;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import java.util.Set;
|
||||
|
||||
global Dictionary dictionary
|
||||
|
||||
query "getEntities"
|
||||
$result: Entity()
|
||||
end
|
||||
|
||||
// --------------------------------------- AI rules -------------------------------------------------------------------
|
||||
rule "-1: find entities from dictionary"
|
||||
rule "0: find entities from entry dictionary"
|
||||
salience 100
|
||||
no-loop true
|
||||
when
|
||||
section: Section(!text.isEmpty())
|
||||
$section: Section()
|
||||
$paragraph: Paragraph(sectionNumber == $section.sectionNumber)
|
||||
$entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition())
|
||||
then
|
||||
section.findDictionaryEntities();
|
||||
update(section);
|
||||
end
|
||||
|
||||
rule "0: Add CBI_author from ai"
|
||||
when
|
||||
section: Section(aiMatchesType("CBI_author"))
|
||||
then
|
||||
section.addAiEntities("CBI_author", "CBI_author");
|
||||
end
|
||||
|
||||
rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
|
||||
when
|
||||
section: Section(aiMatchesType("ORG"))
|
||||
then
|
||||
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
||||
end
|
||||
|
||||
rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
|
||||
when
|
||||
section: Section(aiMatchesType("STREET"))
|
||||
then
|
||||
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
||||
end
|
||||
|
||||
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
|
||||
when
|
||||
section: Section(aiMatchesType("CITY"))
|
||||
then
|
||||
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
|
||||
end
|
||||
|
||||
|
||||
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
||||
|
||||
rule "1: Redact CBI Authors (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
|
||||
then
|
||||
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "2: Redact CBI Authors (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
|
||||
then
|
||||
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "3: Redact not CBI Address (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
|
||||
then
|
||||
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
|
||||
section.ignoreRecommendations("CBI_address");
|
||||
end
|
||||
|
||||
rule "4: Redact CBI Address (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
|
||||
then
|
||||
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "5: Do not redact genitive CBI_author"
|
||||
when
|
||||
section: Section(matchesType("CBI_author"))
|
||||
then
|
||||
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
|
||||
end
|
||||
|
||||
|
||||
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
|
||||
then
|
||||
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
|
||||
then
|
||||
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
|
||||
then
|
||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
|
||||
then
|
||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "16: Add recommendation for Addresses in Test Organism sections"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
|
||||
then
|
||||
section.recommendLineAfter("Source:", "CBI_address");
|
||||
end
|
||||
|
||||
rule "17: Add recommendation for Addresses in Test Animals sections"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
|
||||
then
|
||||
section.recommendLineAfter("Source", "CBI_address");
|
||||
List<EntityPositionSequence> eps = new LinkedList<>();
|
||||
eps.add($entityPositionSequence);
|
||||
Set<Engine> engineSet = new HashSet<>();
|
||||
engineSet.add(Engine.DICTIONARY);
|
||||
Set<Entity> empty = new HashSet<>();
|
||||
insert(new Entity("CBI_author",
|
||||
"custom",
|
||||
true,
|
||||
format("Found in Dictionary: %s", "custom"),
|
||||
eps,
|
||||
$section.getHeadline(),
|
||||
0,
|
||||
$section.getSectionNumber(),
|
||||
$paragraph.getParagraphNumber(),
|
||||
"",
|
||||
true,
|
||||
"",
|
||||
"",
|
||||
-1,
|
||||
-1,
|
||||
false,
|
||||
engineSet,
|
||||
empty,
|
||||
EntityType.ENTITY
|
||||
));
|
||||
end
|
||||
|
||||
/*
|
||||
|
||||
rule "18: Do not redact Names and Addresses if Published Information found"
|
||||
when
|
||||
section: Section(matchesType("published_information"))
|
||||
then
|
||||
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
|
||||
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
|
||||
end
|
||||
|
||||
*/
|
||||
|
||||
// --------------------------------------- PII rules -------------------------------------------------------------------
|
||||
|
||||
|
||||
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "21: Redact Emails by RegEx (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
|
||||
then
|
||||
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "22: Redact Emails by RegEx (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
|
||||
then
|
||||
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "23: Redact contact information (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|
||||
|| text.contains("Contact:")
|
||||
|| text.contains("Alternative contact:")
|
||||
|| (text.contains("No:") && text.contains("Fax"))
|
||||
|| (text.contains("Contact:") && text.contains("Tel.:"))
|
||||
|| text.contains("European contact:")
|
||||
))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "24: Redact contact information (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|
||||
|| text.contains("Contact:")
|
||||
|| text.contains("Alternative contact:")
|
||||
|| (text.contains("No:") && text.contains("Fax"))
|
||||
|| (text.contains("Contact:") && text.contains("Tel.:"))
|
||||
|| text.contains("European contact:")
|
||||
))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
|
||||
text.contains("Contact")
|
||||
|| text.contains("Telephone")
|
||||
|| text.contains("Phone")
|
||||
|| text.contains("Fax")
|
||||
|| text.contains("Tel")
|
||||
|| text.contains("Ter")
|
||||
|| text.contains("Mobile")
|
||||
|| text.contains("Fel")
|
||||
|| text.contains("Fer")
|
||||
))
|
||||
then
|
||||
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
|
||||
text.contains("Contact")
|
||||
|| text.contains("Telephone")
|
||||
|| text.contains("Phone")
|
||||
|| text.contains("Fax")
|
||||
|| text.contains("Tel")
|
||||
|| text.contains("Ter")
|
||||
|| text.contains("Mobile")
|
||||
|| text.contains("Fel")
|
||||
|| text.contains("Fer")
|
||||
))
|
||||
then
|
||||
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("AUTHOR(S):")
|
||||
&& searchText.contains("COMPLETION DATE:")
|
||||
&& !searchText.contains("STUDY COMPLETION DATE:")
|
||||
)
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "28: Redact AUTHOR(S) (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("AUTHOR(S):")
|
||||
&& searchText.contains("COMPLETION DATE:")
|
||||
&& !searchText.contains("STUDY COMPLETION DATE:")
|
||||
)
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("AUTHOR(S):")
|
||||
&& searchText.contains("STUDY COMPLETION DATE:")
|
||||
)
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "30: Redact AUTHOR(S) (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("AUTHOR(S):")
|
||||
&& searchText.contains("STUDY COMPLETION DATE:")
|
||||
)
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("PERFORMING LABORATORY:")
|
||||
)
|
||||
then
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
|
||||
end
|
||||
|
||||
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("PERFORMING LABORATORY:"))
|
||||
then
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "33: Redact study director abbreviation"
|
||||
when
|
||||
section: Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
|
||||
then
|
||||
section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
// --------------------------------------- other rules -------------------------------------------------------------------
|
||||
|
||||
rule "34: Purity Hint"
|
||||
when
|
||||
section: Section(searchText.toLowerCase().contains("purity"))
|
||||
then
|
||||
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
|
||||
end
|
||||
|
||||
|
||||
|
||||
rule "35: Redact signatures (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
|
||||
then
|
||||
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "36: Redact signatures (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
|
||||
then
|
||||
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "43: Redact Logos (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
|
||||
then
|
||||
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "44: Redact Author if contained in paragraph with 'unpublished'"
|
||||
rule "44: Don't redact Author if contained in paragraph with published information"
|
||||
when
|
||||
$section: Section()
|
||||
$entity: Entity(type == "CBI_author") from $section.getEntities()
|
||||
$publishedEntity: Entity(type == "published_information") from $section.getEntities()
|
||||
$paragraph: Paragraph(sectionNumber == $section.sectionNumber)
|
||||
then
|
||||
if ($paragraph.containsEntity($entity) && $paragraph.containsEntity($publishedEntity) ) {
|
||||
$paragraph: Paragraph($section.sectionNumber == sectionNumber)
|
||||
|
||||
$entity.setRedaction(false);
|
||||
}
|
||||
$authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber())
|
||||
$publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber())
|
||||
then
|
||||
$authorEntity.setRedaction(false);
|
||||
end
|
||||
*/
|
||||
Loading…
x
Reference in New Issue
Block a user