RED-6093: Prototype find entities in rules
*added a prototype paragraph rule
This commit is contained in:
Kilian Schuettler
parent
dd1b838a5c
commit
66b2d52d40
@ -1,15 +1,19 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||||
|
|
||||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
|
||||||
|
|
||||||
import lombok.*;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
|
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||||
|
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.EqualsAndHashCode;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@Builder
|
@Builder
|
||||||
@NoArgsConstructor
|
@NoArgsConstructor
|
||||||
@ -38,6 +42,9 @@ public class Entity implements ReasonHolder {
|
|||||||
@EqualsAndHashCode.Include
|
@EqualsAndHashCode.Include
|
||||||
private int sectionNumber;
|
private int sectionNumber;
|
||||||
|
|
||||||
|
@EqualsAndHashCode.Include
|
||||||
|
private int paragraphNumber;
|
||||||
|
|
||||||
private boolean isDictionaryEntry;
|
private boolean isDictionaryEntry;
|
||||||
|
|
||||||
private String textBefore;
|
private String textBefore;
|
||||||
|
|||||||
@ -0,0 +1,61 @@
|
|||||||
|
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||||
|
|
||||||
|
import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
public class Paragraph {
|
||||||
|
|
||||||
|
private final String searchText;
|
||||||
|
private final List<RedTextPosition> textPositions;
|
||||||
|
private final int[] searchTextToPositions;
|
||||||
|
private final int sectionNumber;
|
||||||
|
|
||||||
|
private final int start;
|
||||||
|
private final int end;
|
||||||
|
|
||||||
|
|
||||||
|
public Paragraph(TextBlock textBlock, int sectionNumber, int start, int end) {
|
||||||
|
|
||||||
|
textPositions = textBlock.getSequences().stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
|
||||||
|
this.searchTextToPositions = new int[textPositions.size()];
|
||||||
|
this.searchText = buildString(textBlock.getSequences());
|
||||||
|
this.start = start;
|
||||||
|
this.end = end;
|
||||||
|
this.sectionNumber = sectionNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean containsEntity(Entity entity) {
|
||||||
|
return entity.getStart() >= start && entity.getEnd() <= end;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildSearchText() {
|
||||||
|
|
||||||
|
StringBuilder searchTextBuilder = new StringBuilder();
|
||||||
|
int numberOfNormalCharacters = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < textPositions.size(); ++i) {
|
||||||
|
if (isNormalText(textPositions.get(i).getUnicode())) {
|
||||||
|
searchTextBuilder.append(textPositions.get(i).getUnicode());
|
||||||
|
this.searchTextToPositions[numberOfNormalCharacters] = i;
|
||||||
|
++numberOfNormalCharacters;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return searchTextBuilder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private boolean isNormalText(String unicode) {
|
||||||
|
|
||||||
|
return !unicode.equals("\n") && //
|
||||||
|
!unicode.equals("");
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -27,7 +27,6 @@ import com.iqser.red.service.redaction.v1.model.Engine;
|
|||||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
|
import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
|
||||||
@ -39,7 +38,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
|||||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
|
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
|
||||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||||
|
|
||||||
import io.micrometer.core.annotation.Timed;
|
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@ -56,7 +54,8 @@ public class Section {
|
|||||||
@Builder.Default
|
@Builder.Default
|
||||||
private Map<String, Set<String>> localDictionaryAdds = new HashMap<>();
|
private Map<String, Set<String>> localDictionaryAdds = new HashMap<>();
|
||||||
|
|
||||||
private Set<Entity> entities;
|
@Builder.Default
|
||||||
|
private Set<Entity> entities = new HashSet<>();
|
||||||
|
|
||||||
private Set<Entity> nerEntities;
|
private Set<Entity> nerEntities;
|
||||||
|
|
||||||
@ -97,12 +96,10 @@ public class Section {
|
|||||||
|
|
||||||
private RedactionServiceSettings redactionServiceSettings;
|
private RedactionServiceSettings redactionServiceSettings;
|
||||||
|
|
||||||
|
|
||||||
@Deprecated
|
@Deprecated
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@ThenAction
|
@ThenAction
|
||||||
public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
|
public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
|
||||||
|
|
||||||
redactOrRecommendAiEntities(type, asType, false, 0, null, null);
|
redactOrRecommendAiEntities(type, asType, false, 0, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1775,14 +1772,4 @@ public class Section {
|
|||||||
entities = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
|
entities = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
|
||||||
nerEntities = EntitySearchUtils.clearAndFindPositions(nerEntities, searchableText, dictionary, manualRedactions);
|
nerEntities = EntitySearchUtils.clearAndFindPositions(nerEntities, searchableText, dictionary, manualRedactions);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -1,16 +0,0 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
|
||||||
import lombok.Data;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
@AllArgsConstructor
|
|
||||||
public class SectionSearchableTextPair {
|
|
||||||
|
|
||||||
private Section section;
|
|
||||||
private SearchableText searchableText;
|
|
||||||
private List<Integer> cellStarts;
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,11 +1,13 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||||
|
|
||||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
import java.io.ByteArrayInputStream;
|
||||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
import java.io.InputStream;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.HashMap;
|
||||||
import io.micrometer.core.annotation.Timed;
|
import java.util.HashSet;
|
||||||
import lombok.RequiredArgsConstructor;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.kie.api.KieServices;
|
import org.kie.api.KieServices;
|
||||||
@ -16,11 +18,15 @@ import org.kie.api.runtime.KieContainer;
|
|||||||
import org.kie.api.runtime.KieSession;
|
import org.kie.api.runtime.KieSession;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.ByteArrayInputStream;
|
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||||
import java.io.InputStream;
|
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||||
import java.util.HashMap;
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||||
import java.util.Map;
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||||
|
|
||||||
|
import io.micrometer.core.annotation.Timed;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@ -45,15 +51,16 @@ public class DroolsExecutionService {
|
|||||||
|
|
||||||
|
|
||||||
@Timed("redactmanager_executeRules")
|
@Timed("redactmanager_executeRules")
|
||||||
public Section executeRules(KieContainer kieContainer, Section section) {
|
public List<Section> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
|
||||||
|
|
||||||
|
Set<Entity> entities = new HashSet<>();
|
||||||
KieSession kieSession = kieContainer.newKieSession();
|
KieSession kieSession = kieContainer.newKieSession();
|
||||||
kieSession.setGlobal("section", section);
|
sections.forEach(kieSession::insert);
|
||||||
kieSession.insert(section);
|
paragraphs.forEach(kieSession::insert);
|
||||||
kieSession.fireAllRules();
|
kieSession.fireAllRules();
|
||||||
kieSession.dispose();
|
kieSession.dispose();
|
||||||
|
|
||||||
return section;
|
return sections;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,33 +1,45 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.kie.api.runtime.KieContainer;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
|
||||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
|
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||||
|
|
||||||
import io.micrometer.core.annotation.Timed;
|
import io.micrometer.core.annotation.Timed;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.kie.api.runtime.KieContainer;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import java.util.stream.Stream;
|
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@ -44,7 +56,7 @@ public class EntityRedactionService {
|
|||||||
|
|
||||||
if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
|
if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
|
||||||
|
|
||||||
if(!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
|
if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
|
||||||
//AnalyzeRequest provides immutable list.
|
//AnalyzeRequest provides immutable list.
|
||||||
List<FileAttribute> mergedFileAttributes = new ArrayList<>();
|
List<FileAttribute> mergedFileAttributes = new ArrayList<>();
|
||||||
mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
|
mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
|
||||||
@ -53,7 +65,14 @@ public class EntityRedactionService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
|
Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
|
||||||
FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage, nerEntities);
|
FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
|
||||||
|
dictionary,
|
||||||
|
kieContainer,
|
||||||
|
analyzeRequest,
|
||||||
|
true,
|
||||||
|
hintsPerSectionNumber,
|
||||||
|
imagesPerPage,
|
||||||
|
nerEntities);
|
||||||
EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
|
EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
|
||||||
EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
|
EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
|
||||||
}
|
}
|
||||||
@ -66,15 +85,16 @@ public class EntityRedactionService {
|
|||||||
|
|
||||||
|
|
||||||
public FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
|
public FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
|
||||||
Dictionary dictionary,
|
Dictionary dictionary,
|
||||||
KieContainer kieContainer,
|
KieContainer kieContainer,
|
||||||
AnalyzeRequest analyzeRequest,
|
AnalyzeRequest analyzeRequest,
|
||||||
boolean local,
|
boolean local,
|
||||||
Map<Integer, Set<Entity>> hintsPerSectionNumber,
|
Map<Integer, Set<Entity>> hintsPerSectionNumber,
|
||||||
Map<Integer, Set<Image>> imagesPerPage,
|
Map<Integer, Set<Image>> imagesPerPage,
|
||||||
NerEntities nerEntities) {
|
NerEntities nerEntities) {
|
||||||
|
|
||||||
|
List<Section> sections = new ArrayList<>(reanalysisSections.size());
|
||||||
|
|
||||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
|
||||||
for (SectionText reanalysisSection : reanalysisSections) {
|
for (SectionText reanalysisSection : reanalysisSections) {
|
||||||
|
|
||||||
Set<Entity> nerFound = new HashSet<>();
|
Set<Entity> nerFound = new HashSet<>();
|
||||||
@ -83,8 +103,7 @@ public class EntityRedactionService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
|
log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
|
||||||
|
sections.add(Section.builder()
|
||||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
|
||||||
.isLocal(false)
|
.isLocal(false)
|
||||||
.dictionaryTypes(dictionary.getTypes())
|
.dictionaryTypes(dictionary.getTypes())
|
||||||
.nerEntities(nerFound)
|
.nerEntities(nerFound)
|
||||||
@ -102,25 +121,39 @@ public class EntityRedactionService {
|
|||||||
.isInTable(reanalysisSection.isTable())
|
.isInTable(reanalysisSection.isTable())
|
||||||
.redactionServiceSettings(redactionServiceSettings)
|
.redactionServiceSettings(redactionServiceSettings)
|
||||||
.cellStarts(reanalysisSection.getCellStarts())
|
.cellStarts(reanalysisSection.getCellStarts())
|
||||||
.build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
|
.build());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Set<FileAttribute> addedFileAttributes = new HashSet<>();
|
Set<FileAttribute> addedFileAttributes = new HashSet<>();
|
||||||
Set<Entity> entities = new HashSet<>();
|
Set<Entity> entities = new HashSet<>();
|
||||||
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
|
sections.forEach(section -> {
|
||||||
|
|
||||||
if(!addedFileAttributes.isEmpty()) {
|
if (!addedFileAttributes.isEmpty()) {
|
||||||
//Section.Builder provides immutable list.
|
//Section.Builder provides immutable list.
|
||||||
List<FileAttribute> mergedFileAttributes = new ArrayList<>();
|
List<FileAttribute> mergedFileAttributes = new ArrayList<>();
|
||||||
mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
|
mergedFileAttributes.addAll(section.getAddedFileAttributes());
|
||||||
mergedFileAttributes.addAll(addedFileAttributes);
|
mergedFileAttributes.addAll(addedFileAttributes);
|
||||||
sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
|
section.setFileAttributes(mergedFileAttributes);
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
|
||||||
Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
|
|
||||||
|
|
||||||
|
List<Paragraph> paragraphs = new ArrayList<>();
|
||||||
|
for (var reanalysisSection : reanalysisSections) {
|
||||||
|
int start = 0;
|
||||||
|
int end;
|
||||||
|
for (var textBlock : reanalysisSection.getTextBlocks()) {
|
||||||
|
int length = buildString(textBlock.getSequences()).length();
|
||||||
|
end = start + length;
|
||||||
|
paragraphs.add(new Paragraph(textBlock, reanalysisSection.getSectionNumber(), start, end));
|
||||||
|
start = end + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Section> analysedSections = droolsExecutionService.executeRules(kieContainer, sections, paragraphs, dictionary);
|
||||||
|
|
||||||
|
analysedSections.forEach(analysedSection -> {
|
||||||
addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
|
addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
|
||||||
|
|
||||||
EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
|
EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
|
||||||
@ -130,13 +163,19 @@ public class EntityRedactionService {
|
|||||||
.filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
|
.filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
|
||||||
.collect(Collectors.toSet());
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
|
if (analysedSection.getCellStarts() != null && !analysedSection.getCellStarts().isEmpty()) {
|
||||||
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
|
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
|
||||||
sectionSearchableTextPair.getSearchableText(),
|
analysedSection.getSearchableText(),
|
||||||
dictionary,
|
dictionary,
|
||||||
sectionSearchableTextPair.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
|
analysedSection.getCellStarts(),
|
||||||
|
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
|
||||||
|
redactionServiceSettings.getNumberOfSurroundingWords());
|
||||||
} else {
|
} else {
|
||||||
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
|
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
|
||||||
|
analysedSection.getSearchableText(),
|
||||||
|
dictionary,
|
||||||
|
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
|
||||||
|
redactionServiceSettings.getNumberOfSurroundingWords());
|
||||||
}
|
}
|
||||||
|
|
||||||
entities.addAll(analysedSection.getEntities());
|
entities.addAll(analysedSection.getEntities());
|
||||||
|
|||||||
@ -99,7 +99,7 @@ public final class EntitySearchUtils {
|
|||||||
|
|
||||||
List<Entity> orderedEntities = entitiesByWord.get(word).stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
|
List<Entity> orderedEntities = entitiesByWord.get(word).stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
|
||||||
Entity firstEntity = orderedEntities.get(0);
|
Entity firstEntity = orderedEntities.get(0);
|
||||||
List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord().trim(),
|
List<EntityPositionSequence> positionSequences = text.getSequences(word.trim(),
|
||||||
dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()),
|
dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()),
|
||||||
firstEntity.getTargetSequences());
|
firstEntity.getTargetSequences());
|
||||||
|
|
||||||
|
|||||||
@ -364,7 +364,7 @@ public class RedactionIntegrationTest {
|
|||||||
@Test
|
@Test
|
||||||
public void titleExtraction() throws IOException {
|
public void titleExtraction() throws IOException {
|
||||||
|
|
||||||
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
|
AnalyzeRequest request = prepareStorage("files/Trinexapac/93 Trinexapac-ethyl_RAR_03_Volume_3CA_B-1_2017-03-31.pdf");
|
||||||
|
|
||||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||||
AnalyzeResult result = analyzeService.analyze(request);
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
@ -1111,7 +1111,7 @@ public class RedactionIntegrationTest {
|
|||||||
|
|
||||||
System.out.println("classificationTest");
|
System.out.println("classificationTest");
|
||||||
|
|
||||||
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
|
AnalyzeRequest request = prepareStorage("files/Trinexapac/93 Trinexapac-ethyl_RAR_03_Volume_3CA_B-1_2017-03-31.pdf");
|
||||||
|
|
||||||
RedactionRequest redactionRequest = RedactionRequest.builder()
|
RedactionRequest redactionRequest = RedactionRequest.builder()
|
||||||
.dossierId(request.getDossierId())
|
.dossierId(request.getDossierId())
|
||||||
|
|||||||
@ -1,413 +1,390 @@
|
|||||||
package drools
|
package drools
|
||||||
|
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section
|
import com.iqser.red.service.redaction.v1.server.redaction.model.*
|
||||||
|
|
||||||
global Section section
|
|
||||||
|
|
||||||
|
|
||||||
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
|
||||||
|
|
||||||
//rule "0: Expand CBI Authors with firstname initials"
|
// --------------------------------------- AI rules -------------------------------------------------------------------
|
||||||
// when
|
rule "-1: find entities from dictionary"
|
||||||
// Section(matchesType("CBI_author"))
|
salience 100
|
||||||
// then
|
no-loop true
|
||||||
// section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
|
|
||||||
// end
|
|
||||||
|
|
||||||
rule "-1: Find dictionary entries"
|
|
||||||
when
|
when
|
||||||
Section(findDictionaryEntities());
|
section: Section(!text.isEmpty())
|
||||||
then
|
then
|
||||||
end
|
section.findDictionaryEntities();
|
||||||
|
update(section);
|
||||||
|
end
|
||||||
|
|
||||||
rule "0: Add CBI_author from ai"
|
rule "0: Add CBI_author from ai"
|
||||||
when
|
when
|
||||||
Section(aiMatchesType("CBI_author"))
|
section: Section(aiMatchesType("CBI_author"))
|
||||||
then
|
then
|
||||||
section.addAiEntities("CBI_author", "CBI_author");
|
section.addAiEntities("CBI_author", "CBI_author");
|
||||||
end
|
end
|
||||||
|
|
||||||
rule "0: Combine ai types CBI_author from ai"
|
rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
|
||||||
when
|
when
|
||||||
Section(aiMatchesType("ORG"))
|
section: Section(aiMatchesType("ORG"))
|
||||||
then
|
then
|
||||||
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
||||||
end
|
end
|
||||||
|
|
||||||
rule "0: Expand CBI Authors with firstname initials"
|
rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
|
||||||
when
|
when
|
||||||
Section(matchesType("CBI_author"))
|
section: Section(aiMatchesType("STREET"))
|
||||||
then
|
then
|
||||||
section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
|
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
||||||
end
|
end
|
||||||
|
|
||||||
rule "0: Expand CBI_author and PII matches with salutation prefix"
|
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
|
||||||
when
|
when
|
||||||
Section((matchesType("CBI_author") || matchesType("PII")) && (
|
section: Section(aiMatchesType("CITY"))
|
||||||
searchText.contains("Mr")
|
|
||||||
|| searchText.contains("Mrs")
|
|
||||||
|| searchText.contains("Ms")
|
|
||||||
|| searchText.contains("Miss")
|
|
||||||
|| searchText.contains("Sir")
|
|
||||||
|| searchText.contains("Madam")
|
|
||||||
|| searchText.contains("Madame")
|
|
||||||
|| searchText.contains("Mme")
|
|
||||||
))
|
|
||||||
then
|
then
|
||||||
section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
|
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
|
||||||
section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "1: Redacted because Section contains Vertebrate"
|
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
||||||
|
|
||||||
|
rule "1: Redact CBI Authors (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(matchesType("vertebrate"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
|
||||||
then
|
then
|
||||||
section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
end
|
||||||
|
|
||||||
|
rule "2: Redact CBI Authors (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
|
||||||
|
then
|
||||||
|
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "2: Not Redacted because Section contains no Vertebrate"
|
rule "3: Redact not CBI Address (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(!matchesType("vertebrate"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
|
||||||
then
|
then
|
||||||
section.redactNot("CBI_author", 2, "No Vertebrate found");
|
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
|
||||||
section.redactNot("CBI_address", 2, "No Vertebrate found");
|
section.ignoreRecommendations("CBI_address");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "4: Redact CBI Address (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
|
||||||
|
then
|
||||||
|
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
|
rule "5: Do not redact genitive CBI_author"
|
||||||
when
|
when
|
||||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
|
section: Section(matchesType("CBI_author"))
|
||||||
then
|
then
|
||||||
section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found");
|
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
|
||||||
section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found");
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
|
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
|
||||||
then
|
then
|
||||||
section.redact("CBI_author", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redact("CBI_address", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
end
|
||||||
|
|
||||||
|
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
|
||||||
|
then
|
||||||
|
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "5: Do not redact Names and Addresses if no redaction Indicator is contained"
|
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(matchesType("vertebrate"), matchesType("published_information"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
|
||||||
then
|
then
|
||||||
section.redactNotAndReference("CBI_author","published_information", 5, "Vertebrate and Published Information found");
|
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactNotAndReference("CBI_address","published_information", 5, "Vertebrate and Published Information found");
|
end
|
||||||
|
|
||||||
|
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
|
||||||
|
then
|
||||||
|
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "6: Not redacted because Vertebrate Study = N"
|
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
|
||||||
then
|
then
|
||||||
section.redactNotCell("Author(s)", 6, "CBI_author", true, "Not redacted because row is not a vertebrate study");
|
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactNot("CBI_author", 6, "Not redacted because row is not a vertebrate study");
|
end
|
||||||
section.redactNot("CBI_address", 6, "Not redacted because row is not a vertebrate study");
|
|
||||||
section.highlightCell("Vertebrate study Y/N", 6, "hint_only");
|
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
|
||||||
|
then
|
||||||
|
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
|
||||||
|
then
|
||||||
|
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
|
||||||
|
then
|
||||||
|
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "7: Redact if must redact entry is found"
|
rule "16: Add recommendation for Addresses in Test Organism sections"
|
||||||
when
|
when
|
||||||
Section(matchesType("must_redact"))
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
|
||||||
then
|
then
|
||||||
section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
section.recommendLineAfter("Source:", "CBI_address");
|
||||||
section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
rule "17: Add recommendation for Addresses in Test Animals sections"
|
||||||
rule "8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
|
|
||||||
when
|
when
|
||||||
Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
|
||||||
then
|
then
|
||||||
section.redactCell("Author(s)", 8, "CBI_author", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
section.recommendLineAfter("Source", "CBI_address");
|
||||||
section.redact("CBI_address", 8, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
||||||
section.highlightCell("Vertebrate study Y/N", 8, "must_redact");
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
/*
|
||||||
|
|
||||||
rule "9: Redact sponsor company"
|
rule "18: Do not redact Names and Addresses if Published Information found"
|
||||||
when
|
when
|
||||||
Section(searchText.toLowerCase().contains("batches produced at"))
|
section: Section(matchesType("published_information"))
|
||||||
then
|
then
|
||||||
section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 9, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
|
||||||
section.addHintAnnotation("batches produced at", "must_redact");
|
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
*/
|
||||||
rule "10: Redact determination of residues"
|
|
||||||
when
|
|
||||||
Section((
|
|
||||||
searchText.toLowerCase.contains("determination of residues") ||
|
|
||||||
searchText.toLowerCase.contains("determination of total residues")
|
|
||||||
) && (
|
|
||||||
searchText.toLowerCase.contains("livestock") ||
|
|
||||||
searchText.toLowerCase.contains("live stock") ||
|
|
||||||
searchText.toLowerCase.contains("tissue") ||
|
|
||||||
searchText.toLowerCase.contains("tissues") ||
|
|
||||||
searchText.toLowerCase.contains("liver") ||
|
|
||||||
searchText.toLowerCase.contains("muscle") ||
|
|
||||||
searchText.toLowerCase.contains("bovine") ||
|
|
||||||
searchText.toLowerCase.contains("ruminant") ||
|
|
||||||
searchText.toLowerCase.contains("ruminants")
|
|
||||||
))
|
|
||||||
then
|
|
||||||
section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
||||||
section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
||||||
section.addHintAnnotation("determination of residues", "must_redact");
|
|
||||||
section.addHintAnnotation("livestock", "must_redact");
|
|
||||||
section.addHintAnnotation("live stock", "must_redact");
|
|
||||||
section.addHintAnnotation("tissue", "must_redact");
|
|
||||||
section.addHintAnnotation("tissues", "must_redact");
|
|
||||||
section.addHintAnnotation("liver", "must_redact");
|
|
||||||
section.addHintAnnotation("muscle", "must_redact");
|
|
||||||
section.addHintAnnotation("bovine", "must_redact");
|
|
||||||
section.addHintAnnotation("ruminant", "must_redact");
|
|
||||||
section.addHintAnnotation("ruminants", "must_redact");
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
rule "11: Redact if CTL/* or BL/* was found"
|
|
||||||
when
|
|
||||||
Section(searchText.contains("CTL/") || searchText.contains("BL/"))
|
|
||||||
then
|
|
||||||
section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
||||||
section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
||||||
section.addHintAnnotation("CTL", "must_redact");
|
|
||||||
section.addHintAnnotation("BL", "must_redact");
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
rule "12: Redact and add recommendation for et al. author"
|
|
||||||
when
|
|
||||||
Section(searchText.contains("et al"))
|
|
||||||
then
|
|
||||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
rule "13: Add recommendation for Addresses in Test Organism sections"
|
|
||||||
when
|
|
||||||
Section(searchText.contains("Species:") && searchText.contains("Source:"))
|
|
||||||
then
|
|
||||||
section.recommendLineAfter("Source:", "CBI_address");
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
rule "14: Add recommendation for Addresses in Test Animals sections"
|
|
||||||
when
|
|
||||||
Section(searchText.contains("Species") && searchText.contains("Source"))
|
|
||||||
then
|
|
||||||
section.recommendLineAfter("Source", "CBI_address");
|
|
||||||
end
|
|
||||||
|
|
||||||
// --------------------------------------- PII rules -------------------------------------------------------------------
|
// --------------------------------------- PII rules -------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
rule "14: Redacted PII Personal Identification Information"
|
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(matchesType("PII"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
|
||||||
then
|
then
|
||||||
section.redact("PII", 14, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
|
||||||
|
then
|
||||||
|
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "15: Redact Emails by RegEx"
|
rule "21: Redact Emails by RegEx (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(searchText.contains("@"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
|
||||||
then
|
then
|
||||||
section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 15, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "22: Redact Emails by RegEx (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
|
||||||
|
then
|
||||||
|
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "16: Redact contact information"
|
rule "23: Redact contact information (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(text.contains("Contact point:")
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|
||||||
|| text.contains("Phone:")
|
|| text.contains("Contact:")
|
||||||
|| text.contains("Fax:")
|
|| text.contains("Alternative contact:")
|
||||||
|| text.contains("Tel.:")
|
|| (text.contains("No:") && text.contains("Fax"))
|
||||||
|| text.contains("Tel:")
|
|| (text.contains("Contact:") && text.contains("Tel.:"))
|
||||||
|| text.contains("E-mail:")
|
|| text.contains("European contact:")
|
||||||
|| text.contains("Email:")
|
))
|
||||||
|| text.contains("e-mail:")
|
|
||||||
|| text.contains("E-mail address:")
|
|
||||||
|| text.contains("Alternative contact:")
|
|
||||||
|| text.contains("Telephone number:")
|
|
||||||
|| text.contains("Telephone No:")
|
|
||||||
|| text.contains("Fax number:")
|
|
||||||
|| text.contains("Telephone:")
|
|
||||||
|| text.contains("European contact:"))
|
|
||||||
then
|
then
|
||||||
section.redactLineAfter("Contact point:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Phone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Fax:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Tel:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("E-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Email:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("e-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("E-mail address:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("Contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("Alternative contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("Telephone number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("Telephone No:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("Fax number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("Telephone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactBetween("No:", "Fax", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactBetween("Contact:", "Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("European contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
rule "24: Redact contact information (Vertebrate study)"
|
||||||
rule "17: Redact contact information if applicant is found"
|
|
||||||
when
|
when
|
||||||
Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|
||||||
|
|| text.contains("Contact:")
|
||||||
|
|| text.contains("Alternative contact:")
|
||||||
|
|| (text.contains("No:") && text.contains("Fax"))
|
||||||
|
|| (text.contains("Contact:") && text.contains("Tel.:"))
|
||||||
|
|| text.contains("European contact:")
|
||||||
|
))
|
||||||
then
|
then
|
||||||
section.redactLineAfter("Contact point:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Phone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Fax:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Tel:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("E-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Email:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
end
|
||||||
section.redactLineAfter("e-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("E-mail address:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
|
||||||
section.redactLineAfter("Contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
when
|
||||||
section.redactLineAfter("Alternative contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
|
||||||
section.redactLineAfter("Telephone number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
text.contains("Contact")
|
||||||
section.redactLineAfter("Telephone No:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|| text.contains("Telephone")
|
||||||
section.redactLineAfter("Fax number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|| text.contains("Phone")
|
||||||
section.redactLineAfter("Telephone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|| text.contains("Fax")
|
||||||
section.redactBetween("No:", "Fax", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|| text.contains("Tel")
|
||||||
section.redactBetween("Contact:", "Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|| text.contains("Ter")
|
||||||
section.redactLineAfter("European contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|| text.contains("Mobile")
|
||||||
|
|| text.contains("Fel")
|
||||||
|
|| text.contains("Fer")
|
||||||
|
))
|
||||||
|
then
|
||||||
|
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
|
||||||
|
text.contains("Contact")
|
||||||
|
|| text.contains("Telephone")
|
||||||
|
|| text.contains("Phone")
|
||||||
|
|| text.contains("Fax")
|
||||||
|
|| text.contains("Tel")
|
||||||
|
|| text.contains("Ter")
|
||||||
|
|| text.contains("Mobile")
|
||||||
|
|| text.contains("Fel")
|
||||||
|
|| text.contains("Fer")
|
||||||
|
))
|
||||||
|
then
|
||||||
|
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||||
|
&& searchText.contains("AUTHOR(S):")
|
||||||
|
&& searchText.contains("COMPLETION DATE:")
|
||||||
|
&& !searchText.contains("STUDY COMPLETION DATE:")
|
||||||
|
)
|
||||||
|
then
|
||||||
|
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "28: Redact AUTHOR(S) (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||||
|
&& searchText.contains("AUTHOR(S):")
|
||||||
|
&& searchText.contains("COMPLETION DATE:")
|
||||||
|
&& !searchText.contains("STUDY COMPLETION DATE:")
|
||||||
|
)
|
||||||
|
then
|
||||||
|
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "18: Redact contact information if Producer is found"
|
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||||
|
&& searchText.contains("AUTHOR(S):")
|
||||||
|
&& searchText.contains("STUDY COMPLETION DATE:")
|
||||||
|
)
|
||||||
then
|
then
|
||||||
section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
section.redactLineAfter("Telephone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
end
|
||||||
section.redactLineAfter("Phone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
section.redactLineAfter("Fax:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
rule "30: Redact AUTHOR(S) (Vertebrate study)"
|
||||||
section.redactLineAfter("E-mail:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
when
|
||||||
section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||||
section.redactLineAfter("Fax number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
&& searchText.contains("AUTHOR(S):")
|
||||||
section.redactLineAfter("Telephone number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
&& searchText.contains("STUDY COMPLETION DATE:")
|
||||||
section.redactLineAfter("Tel:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
)
|
||||||
section.redactBetween("No:", "Fax", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
then
|
||||||
|
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "19: Redact AUTHOR(S)"
|
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(searchText.contains("AUTHOR(S):") && fileAttributeByPlaceholderEquals("{fileattributes.vertebrateStudy}", "true"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||||
|
&& searchText.contains("PERFORMING LABORATORY:")
|
||||||
|
)
|
||||||
then
|
then
|
||||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
|
||||||
rule "20: Redact PERFORMING LABORATORY"
|
|
||||||
when
|
when
|
||||||
Section(searchText.contains("PERFORMING LABORATORY:"))
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||||
|
&& searchText.contains("PERFORMING LABORATORY:"))
|
||||||
then
|
then
|
||||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 20, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
rule "33: Redact study director abbreviation"
|
||||||
rule "21: Redact On behalf of Sequani Ltd.:"
|
|
||||||
when
|
when
|
||||||
Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
|
section: Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
|
||||||
then
|
then
|
||||||
section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 21, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "22: Redact On behalf of Syngenta Ltd.:"
|
|
||||||
when
|
|
||||||
Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
|
|
||||||
then
|
|
||||||
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
||||||
end
|
|
||||||
|
|
||||||
// --------------------------------------- other rules -------------------------------------------------------------------
|
// --------------------------------------- other rules -------------------------------------------------------------------
|
||||||
|
|
||||||
rule "25: Redact Purity"
|
rule "34: Purity Hint"
|
||||||
when
|
when
|
||||||
Section(searchText.contains("purity"))
|
section: Section(searchText.toLowerCase().contains("purity"))
|
||||||
then
|
then
|
||||||
section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
|
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "26: Redact signatures"
|
|
||||||
|
rule "35: Redact signatures (Non vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(matchesImageType("signature"))
|
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
|
||||||
then
|
then
|
||||||
section.redactImage("signature", 26, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "36: Redact signatures (Vertebrate study)"
|
||||||
|
when
|
||||||
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
|
||||||
|
then
|
||||||
|
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "27: Redact formula"
|
rule "43: Redact Logos (Vertebrate study)"
|
||||||
when
|
when
|
||||||
Section(matchesImageType("formula"))
|
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
|
||||||
then
|
then
|
||||||
section.redactImage("formula", 27, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
end
|
end
|
||||||
|
|
||||||
|
rule "44: Redact Author if contained in paragraph with 'unpublished'"
|
||||||
rule "28: Redact Logos"
|
|
||||||
when
|
when
|
||||||
Section(matchesImageType("logo"))
|
$section: Section()
|
||||||
|
$entity: Entity(type == "CBI_author") from $section.getEntities()
|
||||||
|
$publishedEntity: Entity(type == "published_information") from $section.getEntities()
|
||||||
|
$paragraph: Paragraph(sectionNumber == $section.sectionNumber)
|
||||||
then
|
then
|
||||||
section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
if ($paragraph.containsEntity($entity) && $paragraph.containsEntity($publishedEntity) ) {
|
||||||
|
|
||||||
|
$entity.setRedaction(false);
|
||||||
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
rule "29: Redact Dossier Redactions"
|
|
||||||
when
|
|
||||||
Section(matchesType("dossier_redactions"))
|
|
||||||
then
|
|
||||||
section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
|
|
||||||
end
|
|
||||||
|
|
||||||
rule "30: Ignore dossier_redactions if confidential"
|
|
||||||
when
|
|
||||||
Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions"));
|
|
||||||
then
|
|
||||||
section.ignore("dossier_redactions");
|
|
||||||
end
|
|
||||||
|
|
||||||
// ex. "New Rules for PAD" - "Annex A" - page 21, page 35 (table without header), page 38 (in-text)
|
|
||||||
// https://www.regexplanet.com/share/index.html?share=yyyypb71xkr
|
|
||||||
rule "101: Redact CAS numbers"
|
|
||||||
when
|
|
||||||
Section(hasTableHeader("Sample #"))
|
|
||||||
then
|
|
||||||
section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
||||||
end
|
|
||||||
|
|
||||||
rule "102: Guidelines FileAttributes"
|
|
||||||
when
|
|
||||||
Section((text.contains("DATA REQUIREMENT(S):") || text.contains("TEST GUIDELINE(S):")) && (text.contains("OECD") || text.contains("EPA") || text.contains("OPPTS")))
|
|
||||||
then
|
|
||||||
section.addFileAttribute("OECD Number", "OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", false, 0);
|
|
||||||
end
|
|
||||||
|
|
||||||
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
|
|
||||||
when
|
|
||||||
Section(hasTableHeader("h5.1"))
|
|
||||||
then
|
|
||||||
section.redactCell("h5.1", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
|
||||||
end
|
|
||||||
Loading…
x
Reference in New Issue
Block a user