RED-6093: Prototype find entities in rules
*added improved string to text position mapping
This commit is contained in:
Kilian Schuettler
Kilian Schuettler
parent
18dc68714b
commit
3aa2f84f08
@ -16,7 +16,6 @@ import java.util.Set;
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Entity implements ReasonHolder {
|
||||
|
||||
private String word;
|
||||
private String type;
|
||||
private boolean redaction;
|
||||
@ -26,6 +25,7 @@ public class Entity implements ReasonHolder {
|
||||
private List<EntityPositionSequence> positionSequences = new ArrayList<>();
|
||||
private List<TextPositionSequence> targetSequences;
|
||||
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer start;
|
||||
@EqualsAndHashCode.Include
|
||||
@ -66,6 +66,7 @@ public class Entity implements ReasonHolder {
|
||||
String headline,
|
||||
int matchedRule,
|
||||
int sectionNumber,
|
||||
int paragraphNumber,
|
||||
String legalBasis,
|
||||
boolean isDictionaryEntry,
|
||||
String textBefore,
|
||||
@ -85,6 +86,7 @@ public class Entity implements ReasonHolder {
|
||||
this.headline = headline;
|
||||
this.matchedRule = matchedRule;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.paragraphNumber = paragraphNumber;
|
||||
this.legalBasis = legalBasis;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.textBefore = textBefore;
|
||||
@ -104,6 +106,7 @@ public class Entity implements ReasonHolder {
|
||||
Integer end,
|
||||
String headline,
|
||||
int sectionNumber,
|
||||
int paragraphNumber,
|
||||
boolean isDictionaryEntry,
|
||||
boolean isDossierDictionaryEntry,
|
||||
Engine engine,
|
||||
@ -115,6 +118,7 @@ public class Entity implements ReasonHolder {
|
||||
this.end = end;
|
||||
this.headline = headline;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.paragraphNumber = paragraphNumber;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.isDossierDictionaryEntry = isDossierDictionaryEntry;
|
||||
this.engines.add(engine);
|
||||
|
||||
@ -1,61 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class Paragraph {
|
||||
|
||||
private final String searchText;
|
||||
private final List<RedTextPosition> textPositions;
|
||||
private final int[] searchTextToPositions;
|
||||
private final int sectionNumber;
|
||||
|
||||
private final int start;
|
||||
private final int end;
|
||||
|
||||
|
||||
public Paragraph(TextBlock textBlock, int sectionNumber, int start, int end) {
|
||||
|
||||
textPositions = textBlock.getSequences().stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
|
||||
this.searchTextToPositions = new int[textPositions.size()];
|
||||
this.searchText = buildString(textBlock.getSequences());
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.sectionNumber = sectionNumber;
|
||||
}
|
||||
|
||||
public boolean containsEntity(Entity entity) {
|
||||
return entity.getStart() >= start && entity.getEnd() <= end;
|
||||
}
|
||||
|
||||
private String buildSearchText() {
|
||||
|
||||
StringBuilder searchTextBuilder = new StringBuilder();
|
||||
int numberOfNormalCharacters = 0;
|
||||
|
||||
for (int i = 0; i < textPositions.size(); ++i) {
|
||||
if (isNormalText(textPositions.get(i).getUnicode())) {
|
||||
searchTextBuilder.append(textPositions.get(i).getUnicode());
|
||||
this.searchTextToPositions[numberOfNormalCharacters] = i;
|
||||
++numberOfNormalCharacters;
|
||||
}
|
||||
}
|
||||
return searchTextBuilder.toString();
|
||||
}
|
||||
|
||||
|
||||
private boolean isNormalText(String unicode) {
|
||||
|
||||
return !unicode.equals("\n") && //
|
||||
!unicode.equals("");
|
||||
|
||||
}
|
||||
SearchTextWithTextPositionModel searchTextToTextPosition;
|
||||
int sectionNumber;
|
||||
int paragraphNumber;
|
||||
}
|
||||
|
||||
@ -0,0 +1,22 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Builder
|
||||
@Getter
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class SearchTextWithTextPositionModel {
|
||||
|
||||
String searchText;
|
||||
Map<Integer, Integer> stringCoordsToTextPositionCoords;
|
||||
List<Integer> lineBreaksStringCoords;
|
||||
List<TextPositionSequence> textPositionSequences;
|
||||
}
|
||||
@ -1,17 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
public class SearchableText {
|
||||
@ -184,30 +185,13 @@ public class SearchableText {
|
||||
return stringRepresentation;
|
||||
}
|
||||
|
||||
|
||||
// public List<String> getParagraphStrings(){
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
public static String buildString(List<TextPositionSequence> sequences) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : sequences) {
|
||||
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
sb.append(' ');
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
|
||||
}
|
||||
|
||||
|
||||
@ -1363,6 +1363,7 @@ public class Section {
|
||||
value.getRowSpanStart() + word.length(),
|
||||
headline,
|
||||
sectionNumber,
|
||||
-1,
|
||||
false,
|
||||
false,
|
||||
Engine.RULE,
|
||||
|
||||
@ -4,7 +4,7 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
@ -22,6 +22,8 @@ import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.kie.api.runtime.KieSession;
|
||||
import org.kie.api.runtime.rule.QueryResults;
|
||||
import org.kie.api.runtime.rule.QueryResultsRow;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
@ -88,19 +90,29 @@ public class DroolsExecutionService {
|
||||
|
||||
|
||||
@Timed("redactmanager_executeRules")
|
||||
public List<Section> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
|
||||
public List<Entity> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
KieSession kieSession = kieContainer.newKieSession();
|
||||
kieSession.setGlobal("dictionary", dictionary);
|
||||
sections.forEach(kieSession::insert);
|
||||
paragraphs.forEach(kieSession::insert);
|
||||
kieSession.fireAllRules();
|
||||
List<Entity> entities = getEntities(kieSession);
|
||||
kieSession.dispose();
|
||||
|
||||
return sections;
|
||||
return entities;
|
||||
|
||||
}
|
||||
|
||||
public List<Entity> getEntities(KieSession ks) {
|
||||
List<Entity> entities = new LinkedList<>();
|
||||
QueryResults entitiesResult = ks.getQueryResults("getEntities");
|
||||
for (QueryResultsRow resultsRow : entitiesResult) {
|
||||
entities.add((Entity) resultsRow.get("$result"));
|
||||
}
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
public KieContainer updateRules(String dossierTemplateId) {
|
||||
|
||||
|
||||
@ -0,0 +1,81 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
|
||||
|
||||
@Service
|
||||
public class SearchTextWithTextPositionFactory {
|
||||
|
||||
public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
|
||||
|
||||
Map<Integer, Integer> stringIdxToPositionIdx = new HashMap<>();
|
||||
List<Integer> lineBreaksStringIdx = new LinkedList<>();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
int stringIdx = 0;
|
||||
int positionIdx = 0;
|
||||
|
||||
String currentUnicode;
|
||||
String previousUnicode = " ";
|
||||
|
||||
for (TextPositionSequence word : sequences) {
|
||||
for (int i = 0; i < word.getTextPositions().size(); ++i) {
|
||||
|
||||
currentUnicode = word.getTextPositions().get(i).getUnicode();
|
||||
|
||||
if (isLineBreak(currentUnicode)) {
|
||||
lineBreaksStringIdx.add(stringIdx);
|
||||
} else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && //
|
||||
!isHyphenLinebreak(currentUnicode)) {
|
||||
|
||||
sb.append(currentUnicode);
|
||||
stringIdxToPositionIdx.put(stringIdx, positionIdx);
|
||||
++stringIdx;
|
||||
}
|
||||
|
||||
previousUnicode = currentUnicode;
|
||||
++positionIdx;
|
||||
}
|
||||
|
||||
previousUnicode = " ";
|
||||
sb.append(previousUnicode);
|
||||
stringIdxToPositionIdx.put(stringIdx, positionIdx);
|
||||
++stringIdx;
|
||||
}
|
||||
|
||||
return SearchTextWithTextPositionModel.builder()
|
||||
.searchText(sb.toString().stripTrailing())
|
||||
.lineBreaksStringCoords(lineBreaksStringIdx)
|
||||
.stringCoordsToTextPositionCoords(stringIdxToPositionIdx)
|
||||
.textPositionSequences(sequences)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private boolean isLineBreak(String currentUnicode) {
|
||||
|
||||
return Objects.equals(currentUnicode, "\n");
|
||||
}
|
||||
|
||||
|
||||
private boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) {
|
||||
|
||||
return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " ");
|
||||
}
|
||||
|
||||
|
||||
private boolean isHyphenLinebreak(String unicodeCharacter) {
|
||||
|
||||
return unicodeCharacter.matches("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,10 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import org.ahocorasick.trie.Trie;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
@ -12,6 +7,10 @@ import java.util.Locale;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.ahocorasick.trie.Trie;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class SearchImplementation {
|
||||
|
||||
@ -94,13 +93,7 @@ public class SearchImplementation {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public static class MatchPosition {
|
||||
|
||||
private int startIndex;
|
||||
private int endIndex;
|
||||
public record MatchPosition(int startIndex, int endIndex) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -17,8 +17,8 @@ public class EntitySearchUtilsTest {
|
||||
public void testNestedEntitiesRemoval() {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
entities.add(nested);
|
||||
entities.add(nesting);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
@ -40,14 +40,14 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntities1);
|
||||
foundEntities.add(foundEntities2);
|
||||
|
||||
@ -73,14 +73,14 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntities1);
|
||||
foundEntities.add(foundEntities2);
|
||||
|
||||
@ -105,14 +105,14 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntities1);
|
||||
foundEntities.add(foundEntities2);
|
||||
|
||||
@ -137,15 +137,15 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntities1);
|
||||
foundEntities.add(foundEntities2);
|
||||
foundEntities.add(foundEntities3);
|
||||
@ -171,16 +171,16 @@ public class EntitySearchUtilsTest {
|
||||
|
||||
// Arrange
|
||||
Set<Entity> existingEntities = new HashSet<>();
|
||||
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
existingEntities.add(existingEntity1);
|
||||
existingEntities.add(existingEntity2);
|
||||
|
||||
Set<Entity> foundEntities = new HashSet<>();
|
||||
Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
|
||||
foundEntities.add(foundEntitiesOverlap1);
|
||||
foundEntities.add(foundEntitiesOverlap2);
|
||||
foundEntities.add(foundEntitiesSubset1);
|
||||
|
||||
@ -1,390 +1,67 @@
|
||||
package drools
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.LinkedList;
|
||||
import java.util.HashSet;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import java.util.Set;
|
||||
|
||||
global Dictionary dictionary
|
||||
|
||||
query "getEntities"
|
||||
$result: Entity()
|
||||
end
|
||||
|
||||
// --------------------------------------- AI rules -------------------------------------------------------------------
|
||||
rule "-1: find entities from dictionary"
|
||||
rule "0: find entities from entry dictionary"
|
||||
salience 100
|
||||
no-loop true
|
||||
when
|
||||
section: Section(!text.isEmpty())
|
||||
$section: Section()
|
||||
$paragraph: Paragraph(sectionNumber == $section.sectionNumber)
|
||||
$entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition())
|
||||
then
|
||||
section.findDictionaryEntities();
|
||||
update(section);
|
||||
end
|
||||
|
||||
rule "0: Add CBI_author from ai"
|
||||
when
|
||||
section: Section(aiMatchesType("CBI_author"))
|
||||
then
|
||||
section.addAiEntities("CBI_author", "CBI_author");
|
||||
end
|
||||
|
||||
rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
|
||||
when
|
||||
section: Section(aiMatchesType("ORG"))
|
||||
then
|
||||
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
||||
end
|
||||
|
||||
rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
|
||||
when
|
||||
section: Section(aiMatchesType("STREET"))
|
||||
then
|
||||
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
||||
end
|
||||
|
||||
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
|
||||
when
|
||||
section: Section(aiMatchesType("CITY"))
|
||||
then
|
||||
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
|
||||
end
|
||||
|
||||
|
||||
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
||||
|
||||
rule "1: Redact CBI Authors (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
|
||||
then
|
||||
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "2: Redact CBI Authors (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
|
||||
then
|
||||
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "3: Redact not CBI Address (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
|
||||
then
|
||||
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
|
||||
section.ignoreRecommendations("CBI_address");
|
||||
end
|
||||
|
||||
rule "4: Redact CBI Address (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
|
||||
then
|
||||
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "5: Do not redact genitive CBI_author"
|
||||
when
|
||||
section: Section(matchesType("CBI_author"))
|
||||
then
|
||||
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
|
||||
end
|
||||
|
||||
|
||||
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
|
||||
then
|
||||
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
|
||||
then
|
||||
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
|
||||
then
|
||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
|
||||
then
|
||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "16: Add recommendation for Addresses in Test Organism sections"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
|
||||
then
|
||||
section.recommendLineAfter("Source:", "CBI_address");
|
||||
end
|
||||
|
||||
rule "17: Add recommendation for Addresses in Test Animals sections"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
|
||||
then
|
||||
section.recommendLineAfter("Source", "CBI_address");
|
||||
List<EntityPositionSequence> eps = new LinkedList<>();
|
||||
eps.add($entityPositionSequence);
|
||||
Set<Engine> engineSet = new HashSet<>();
|
||||
engineSet.add(Engine.DICTIONARY);
|
||||
Set<Entity> empty = new HashSet<>();
|
||||
insert(new Entity("CBI_author",
|
||||
"custom",
|
||||
true,
|
||||
format("Found in Dictionary: %s", "custom"),
|
||||
eps,
|
||||
$section.getHeadline(),
|
||||
0,
|
||||
$section.getSectionNumber(),
|
||||
$paragraph.getParagraphNumber(),
|
||||
"",
|
||||
true,
|
||||
"",
|
||||
"",
|
||||
-1,
|
||||
-1,
|
||||
false,
|
||||
engineSet,
|
||||
empty,
|
||||
EntityType.ENTITY
|
||||
));
|
||||
end
|
||||
|
||||
/*
|
||||
|
||||
rule "18: Do not redact Names and Addresses if Published Information found"
|
||||
when
|
||||
section: Section(matchesType("published_information"))
|
||||
then
|
||||
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
|
||||
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
|
||||
end
|
||||
|
||||
*/
|
||||
|
||||
// --------------------------------------- PII rules -------------------------------------------------------------------
|
||||
|
||||
|
||||
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "21: Redact Emails by RegEx (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
|
||||
then
|
||||
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "22: Redact Emails by RegEx (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
|
||||
then
|
||||
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "23: Redact contact information (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|
||||
|| text.contains("Contact:")
|
||||
|| text.contains("Alternative contact:")
|
||||
|| (text.contains("No:") && text.contains("Fax"))
|
||||
|| (text.contains("Contact:") && text.contains("Tel.:"))
|
||||
|| text.contains("European contact:")
|
||||
))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "24: Redact contact information (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|
||||
|| text.contains("Contact:")
|
||||
|| text.contains("Alternative contact:")
|
||||
|| (text.contains("No:") && text.contains("Fax"))
|
||||
|| (text.contains("Contact:") && text.contains("Tel.:"))
|
||||
|| text.contains("European contact:")
|
||||
))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
|
||||
text.contains("Contact")
|
||||
|| text.contains("Telephone")
|
||||
|| text.contains("Phone")
|
||||
|| text.contains("Fax")
|
||||
|| text.contains("Tel")
|
||||
|| text.contains("Ter")
|
||||
|| text.contains("Mobile")
|
||||
|| text.contains("Fel")
|
||||
|| text.contains("Fer")
|
||||
))
|
||||
then
|
||||
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
|
||||
text.contains("Contact")
|
||||
|| text.contains("Telephone")
|
||||
|| text.contains("Phone")
|
||||
|| text.contains("Fax")
|
||||
|| text.contains("Tel")
|
||||
|| text.contains("Ter")
|
||||
|| text.contains("Mobile")
|
||||
|| text.contains("Fel")
|
||||
|| text.contains("Fer")
|
||||
))
|
||||
then
|
||||
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("AUTHOR(S):")
|
||||
&& searchText.contains("COMPLETION DATE:")
|
||||
&& !searchText.contains("STUDY COMPLETION DATE:")
|
||||
)
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "28: Redact AUTHOR(S) (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("AUTHOR(S):")
|
||||
&& searchText.contains("COMPLETION DATE:")
|
||||
&& !searchText.contains("STUDY COMPLETION DATE:")
|
||||
)
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("AUTHOR(S):")
|
||||
&& searchText.contains("STUDY COMPLETION DATE:")
|
||||
)
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "30: Redact AUTHOR(S) (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("AUTHOR(S):")
|
||||
&& searchText.contains("STUDY COMPLETION DATE:")
|
||||
)
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("PERFORMING LABORATORY:")
|
||||
)
|
||||
then
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
|
||||
end
|
||||
|
||||
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
|
||||
&& searchText.contains("PERFORMING LABORATORY:"))
|
||||
then
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "33: Redact study director abbreviation"
|
||||
when
|
||||
section: Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
|
||||
then
|
||||
section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
// --------------------------------------- other rules -------------------------------------------------------------------
|
||||
|
||||
rule "34: Purity Hint"
|
||||
when
|
||||
section: Section(searchText.toLowerCase().contains("purity"))
|
||||
then
|
||||
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
|
||||
end
|
||||
|
||||
|
||||
|
||||
rule "35: Redact signatures (Non vertebrate study)"
|
||||
when
|
||||
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
|
||||
then
|
||||
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "36: Redact signatures (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
|
||||
then
|
||||
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
|
||||
rule "43: Redact Logos (Vertebrate study)"
|
||||
when
|
||||
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
|
||||
then
|
||||
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "44: Redact Author if contained in paragraph with 'unpublished'"
|
||||
rule "44: Don't redact Author if contained in paragraph with published information"
|
||||
when
|
||||
$section: Section()
|
||||
$entity: Entity(type == "CBI_author") from $section.getEntities()
|
||||
$publishedEntity: Entity(type == "published_information") from $section.getEntities()
|
||||
$paragraph: Paragraph(sectionNumber == $section.sectionNumber)
|
||||
then
|
||||
if ($paragraph.containsEntity($entity) && $paragraph.containsEntity($publishedEntity) ) {
|
||||
$paragraph: Paragraph($section.sectionNumber == sectionNumber)
|
||||
|
||||
$entity.setRedaction(false);
|
||||
}
|
||||
$authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber())
|
||||
$publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber())
|
||||
then
|
||||
$authorEntity.setRedaction(false);
|
||||
end
|
||||
*/
|
||||
Loading…
x
Reference in New Issue
Block a user