Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6093: Prototype find entities in rules

Browse Source 
*added improved string to text position mapping
This commit is contained in:
Kilian Schuettler 2023-02-17 17:06:00 +01:00
parent 40fb6a42d7
commit c10ea6d937
10 changed files with 221 additions and 494 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java
View File

@ -16,7 +16,6 @@ import java.util.Set;
@AllArgsConstructor @AllArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true) @EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Entity implements ReasonHolder { public class Entity implements ReasonHolder {
private String word; private String word;
private String type; private String type;
private boolean redaction; private boolean redaction;
@ -26,6 +25,7 @@ public class Entity implements ReasonHolder {
private List<EntityPositionSequence> positionSequences = new ArrayList<>(); private List<EntityPositionSequence> positionSequences = new ArrayList<>();
private List<TextPositionSequence> targetSequences; private List<TextPositionSequence> targetSequences;
@EqualsAndHashCode.Include @EqualsAndHashCode.Include
private Integer start; private Integer start;
@EqualsAndHashCode.Include @EqualsAndHashCode.Include
@ -66,6 +66,7 @@ public class Entity implements ReasonHolder {
String headline, String headline,
int matchedRule, int matchedRule,
int sectionNumber, int sectionNumber,
int paragraphNumber,
String legalBasis, String legalBasis,
boolean isDictionaryEntry, boolean isDictionaryEntry,
String textBefore, String textBefore,
@ -85,6 +86,7 @@ public class Entity implements ReasonHolder {
this.headline = headline; this.headline = headline;
this.matchedRule = matchedRule; this.matchedRule = matchedRule;
this.sectionNumber = sectionNumber; this.sectionNumber = sectionNumber;
this.paragraphNumber = paragraphNumber;
this.legalBasis = legalBasis; this.legalBasis = legalBasis;
this.isDictionaryEntry = isDictionaryEntry; this.isDictionaryEntry = isDictionaryEntry;
this.textBefore = textBefore; this.textBefore = textBefore;
@ -104,6 +106,7 @@ public class Entity implements ReasonHolder {
Integer end, Integer end,
String headline, String headline,
int sectionNumber, int sectionNumber,
int paragraphNumber,
boolean isDictionaryEntry, boolean isDictionaryEntry,
boolean isDossierDictionaryEntry, boolean isDossierDictionaryEntry,
Engine engine, Engine engine,
@ -115,6 +118,7 @@ public class Entity implements ReasonHolder {
this.end = end; this.end = end;
this.headline = headline; this.headline = headline;
this.sectionNumber = sectionNumber; this.sectionNumber = sectionNumber;
this.paragraphNumber = paragraphNumber;
this.isDictionaryEntry = isDictionaryEntry; this.isDictionaryEntry = isDictionaryEntry;
this.isDossierDictionaryEntry = isDossierDictionaryEntry; this.isDossierDictionaryEntry = isDossierDictionaryEntry;
this.engines.add(engine); this.engines.add(engine);

61
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java
View File

@ -1,61 +1,16 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model;
import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString; import lombok.AccessLevel;
import lombok.Builder;
import java.util.List;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import lombok.Getter; import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Getter @Getter
@Builder
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class Paragraph { public class Paragraph {
private final String searchText; SearchTextWithTextPositionModel searchTextToTextPosition;
private final List<RedTextPosition> textPositions; int sectionNumber;
private final int[] searchTextToPositions; int paragraphNumber;
private final int sectionNumber;
private final int start;
private final int end;
public Paragraph(TextBlock textBlock, int sectionNumber, int start, int end) {
textPositions = textBlock.getSequences().stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
this.searchTextToPositions = new int[textPositions.size()];
this.searchText = buildString(textBlock.getSequences());
this.start = start;
this.end = end;
this.sectionNumber = sectionNumber;
}
public boolean containsEntity(Entity entity) {
return entity.getStart() >= start && entity.getEnd() <= end;
}
private String buildSearchText() {
StringBuilder searchTextBuilder = new StringBuilder();
int numberOfNormalCharacters = 0;
for (int i = 0; i < textPositions.size(); ++i) {
if (isNormalText(textPositions.get(i).getUnicode())) {
searchTextBuilder.append(textPositions.get(i).getUnicode());
this.searchTextToPositions[numberOfNormalCharacters] = i;
++numberOfNormalCharacters;
}
}
return searchTextBuilder.toString();
}
private boolean isNormalText(String unicode) {
return !unicode.equals("\n") && //
!unicode.equals("");
}
} }

22
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java Normal file
View File

@ -0,0 +1,22 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.List;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import lombok.AccessLevel;
import lombok.Builder;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Builder
@Getter
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class SearchTextWithTextPositionModel {
String searchText;
Map<Integer, Integer> stringCoordsToTextPositionCoords;
List<Integer> lineBreaksStringCoords;
List<TextPositionSequence> textPositionSequences;
}

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java
View File

@ -1,5 +1,11 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import com.dslplatform.json.JsonAttribute; import com.dslplatform.json.JsonAttribute;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@ -7,12 +13,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import lombok.Getter; import lombok.Getter;
public class SearchableText { public class SearchableText {
@ -186,30 +186,13 @@ public class SearchableText {
return stringRepresentation; return stringRepresentation;
} }
// public List<String> getParagraphStrings(){
//
// }
public static String buildString(List<TextPositionSequence> sequences) { public static String buildString(List<TextPositionSequence> sequences) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
TextPositionSequence previous = null;
for (TextPositionSequence word : sequences) { for (TextPositionSequence word : sequences) {
sb.append(word.toString());
if (previous != null) {
if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
sb.append('\n');
} else {
sb.append(' '); sb.append(' ');
} }
}
sb.append(word.toString());
previous = word;
}
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " "); return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
} }

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -1363,6 +1363,7 @@ public class Section {
value.getRowSpanStart() + word.length(), value.getRowSpanStart() + word.length(),
headline, headline,
sectionNumber, sectionNumber,
-1,
false, false,
false, false,
Engine.RULE, Engine.RULE,

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -4,10 +4,9 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream; import java.io.InputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.kie.api.KieServices; import org.kie.api.KieServices;
@ -16,6 +15,8 @@ import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule; import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieContainer;
import org.kie.api.runtime.KieSession; import org.kie.api.runtime.KieSession;
import org.kie.api.runtime.rule.QueryResults;
import org.kie.api.runtime.rule.QueryResultsRow;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@ -51,19 +52,29 @@ public class DroolsExecutionService {
@Timed("redactmanager_executeRules") @Timed("redactmanager_executeRules")
public List<Section> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) { public List<Entity> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
Set<Entity> entities = new HashSet<>();
KieSession kieSession = kieContainer.newKieSession(); KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("dictionary", dictionary);
sections.forEach(kieSession::insert); sections.forEach(kieSession::insert);
paragraphs.forEach(kieSession::insert); paragraphs.forEach(kieSession::insert);
kieSession.fireAllRules(); kieSession.fireAllRules();
List<Entity> entities = getEntities(kieSession);
kieSession.dispose(); kieSession.dispose();
return sections; return entities;
} }
public List<Entity> getEntities(KieSession ks) {
List<Entity> entities = new LinkedList<>();
QueryResults entitiesResult = ks.getQueryResults("getEntities");
for (QueryResultsRow resultsRow : entitiesResult) {
entities.add((Entity) resultsRow.get("$result"));
}
return entities;
}
public KieContainer updateRules(String dossierTemplateId) { public KieContainer updateRules(String dossierTemplateId) {

81
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java Normal file
View File

@ -0,0 +1,81 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
@Service
public class SearchTextWithTextPositionFactory {
public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
Map<Integer, Integer> stringIdxToPositionIdx = new HashMap<>();
List<Integer> lineBreaksStringIdx = new LinkedList<>();
StringBuilder sb = new StringBuilder();
int stringIdx = 0;
int positionIdx = 0;
String currentUnicode;
String previousUnicode = " ";
for (TextPositionSequence word : sequences) {
for (int i = 0; i < word.getTextPositions().size(); ++i) {
currentUnicode = word.getTextPositions().get(i).getUnicode();
if (isLineBreak(currentUnicode)) {
lineBreaksStringIdx.add(stringIdx);
} else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && //
!isHyphenLinebreak(currentUnicode)) {
sb.append(currentUnicode);
stringIdxToPositionIdx.put(stringIdx, positionIdx);
++stringIdx;
}
previousUnicode = currentUnicode;
++positionIdx;
}
previousUnicode = " ";
sb.append(previousUnicode);
stringIdxToPositionIdx.put(stringIdx, positionIdx);
++stringIdx;
}
return SearchTextWithTextPositionModel.builder()
.searchText(sb.toString().stripTrailing())
.lineBreaksStringCoords(lineBreaksStringIdx)
.stringCoordsToTextPositionCoords(stringIdxToPositionIdx)
.textPositionSequences(sequences)
.build();
}
private boolean isLineBreak(String currentUnicode) {
return Objects.equals(currentUnicode, "\n");
}
private boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) {
return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " ");
}
private boolean isHyphenLinebreak(String unicodeCharacter) {
return unicodeCharacter.matches("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R");
}
}

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java
View File

@ -1,10 +1,5 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils; package com.iqser.red.service.redaction.v1.server.redaction.utils;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.ahocorasick.trie.Trie;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
@ -12,6 +7,10 @@ import java.util.Locale;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.ahocorasick.trie.Trie;
import lombok.Data;
@Data @Data
public class SearchImplementation { public class SearchImplementation {
@ -94,13 +93,7 @@ public class SearchImplementation {
} }
} }
public record MatchPosition(int startIndex, int endIndex) {
@Data
@AllArgsConstructor
public static class MatchPosition {
private int startIndex;
private int endIndex;
} }

50
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java
View File

@ -17,8 +17,8 @@ public class EntitySearchUtilsTest {
public void testNestedEntitiesRemoval() { public void testNestedEntitiesRemoval() {
Set<Entity> entities = new HashSet<>(); Set<Entity> entities = new HashSet<>();
Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
entities.add(nested); entities.add(nested);
entities.add(nesting); entities.add(nesting);
EntitySearchUtils.removeEntitiesContainedInLarger(entities); EntitySearchUtils.removeEntitiesContainedInLarger(entities);
@ -40,14 +40,14 @@ public class EntitySearchUtilsTest {
// Arrange // Arrange
Set<Entity> existingEntities = new HashSet<>(); Set<Entity> existingEntities = new HashSet<>();
Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
existingEntities.add(existingEntity1); existingEntities.add(existingEntity1);
existingEntities.add(existingEntity2); existingEntities.add(existingEntity2);
Set<Entity> foundEntities = new HashSet<>(); Set<Entity> foundEntities = new HashSet<>();
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
foundEntities.add(foundEntities1); foundEntities.add(foundEntities1);
foundEntities.add(foundEntities2); foundEntities.add(foundEntities2);
@ -73,14 +73,14 @@ public class EntitySearchUtilsTest {
// Arrange // Arrange
Set<Entity> existingEntities = new HashSet<>(); Set<Entity> existingEntities = new HashSet<>();
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
existingEntities.add(existingEntity1); existingEntities.add(existingEntity1);
existingEntities.add(existingEntity2); existingEntities.add(existingEntity2);
Set<Entity> foundEntities = new HashSet<>(); Set<Entity> foundEntities = new HashSet<>();
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
foundEntities.add(foundEntities1); foundEntities.add(foundEntities1);
foundEntities.add(foundEntities2); foundEntities.add(foundEntities2);
@ -105,14 +105,14 @@ public class EntitySearchUtilsTest {
// Arrange // Arrange
Set<Entity> existingEntities = new HashSet<>(); Set<Entity> existingEntities = new HashSet<>();
Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
existingEntities.add(existingEntity1); existingEntities.add(existingEntity1);
existingEntities.add(existingEntity2); existingEntities.add(existingEntity2);
Set<Entity> foundEntities = new HashSet<>(); Set<Entity> foundEntities = new HashSet<>();
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
foundEntities.add(foundEntities1); foundEntities.add(foundEntities1);
foundEntities.add(foundEntities2); foundEntities.add(foundEntities2);
@ -137,15 +137,15 @@ public class EntitySearchUtilsTest {
// Arrange // Arrange
Set<Entity> existingEntities = new HashSet<>(); Set<Entity> existingEntities = new HashSet<>();
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
existingEntities.add(existingEntity1); existingEntities.add(existingEntity1);
existingEntities.add(existingEntity2); existingEntities.add(existingEntity2);
Set<Entity> foundEntities = new HashSet<>(); Set<Entity> foundEntities = new HashSet<>();
Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
foundEntities.add(foundEntities1); foundEntities.add(foundEntities1);
foundEntities.add(foundEntities2); foundEntities.add(foundEntities2);
foundEntities.add(foundEntities3); foundEntities.add(foundEntities3);
@ -171,16 +171,16 @@ public class EntitySearchUtilsTest {
// Arrange // Arrange
Set<Entity> existingEntities = new HashSet<>(); Set<Entity> existingEntities = new HashSet<>();
Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
existingEntities.add(existingEntity1); existingEntities.add(existingEntity1);
existingEntities.add(existingEntity2); existingEntities.add(existingEntity2);
Set<Entity> foundEntities = new HashSet<>(); Set<Entity> foundEntities = new HashSet<>();
Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
foundEntities.add(foundEntitiesOverlap1); foundEntities.add(foundEntitiesOverlap1);
foundEntities.add(foundEntitiesOverlap2); foundEntities.add(foundEntitiesOverlap2);
foundEntities.add(foundEntitiesSubset1); foundEntities.add(foundEntitiesSubset1);

425
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1,390 +1,67 @@
package drools package drools
import com.iqser.red.service.redaction.v1.server.redaction.model.* import static java.lang.String.format;
import java.util.List;
import java.util.LinkedList;
import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import java.util.Set;
global Dictionary dictionary
query "getEntities"
$result: Entity()
end
// --------------------------------------- AI rules ------------------------------------------------------------------- // --------------------------------------- AI rules -------------------------------------------------------------------
rule "-1: find entities from dictionary" rule "0: find entities from entry dictionary"
salience 100 salience 100
no-loop true
when when
section: Section(!text.isEmpty()) $section: Section()
$paragraph: Paragraph(sectionNumber == $section.sectionNumber)
$entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition())
then then
section.findDictionaryEntities(); List<EntityPositionSequence> eps = new LinkedList<>();
update(section); eps.add($entityPositionSequence);
end Set<Engine> engineSet = new HashSet<>();
engineSet.add(Engine.DICTIONARY);
rule "0: Add CBI_author from ai" Set<Entity> empty = new HashSet<>();
when insert(new Entity("CBI_author",
section: Section(aiMatchesType("CBI_author")) "custom",
then true,
section.addAiEntities("CBI_author", "CBI_author"); format("Found in Dictionary: %s", "custom"),
end eps,
$section.getHeadline(),
rule "0: Combine address parts from ai to CBI_address (org is mandatory)" 0,
when $section.getSectionNumber(),
section: Section(aiMatchesType("ORG")) $paragraph.getParagraphNumber(),
then "",
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); true,
end "",
"",
rule "0: Combine address parts from ai to CBI_address (street is mandatory)" -1,
when -1,
section: Section(aiMatchesType("STREET")) false,
then engineSet,
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); empty,
end EntityType.ENTITY
));
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
when
section: Section(aiMatchesType("CITY"))
then
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "1: Redact CBI Authors (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
then
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "2: Redact CBI Authors (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
then
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "3: Redact not CBI Address (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
then
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
section.ignoreRecommendations("CBI_address");
end
rule "4: Redact CBI Address (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
then
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "5: Do not redact genitive CBI_author"
when
section: Section(matchesType("CBI_author"))
then
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
end
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "16: Add recommendation for Addresses in Test Organism sections"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
then
section.recommendLineAfter("Source:", "CBI_address");
end
rule "17: Add recommendation for Addresses in Test Animals sections"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
then
section.recommendLineAfter("Source", "CBI_address");
end end
/* /*
rule "44: Don't redact Author if contained in paragraph with published information"
rule "18: Do not redact Names and Addresses if Published Information found"
when
section: Section(matchesType("published_information"))
then
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
end
*/
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
then
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
then
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "21: Redact Emails by RegEx (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "22: Redact Emails by RegEx (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "23: Redact contact information (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|| text.contains("Contact:")
|| text.contains("Alternative contact:")
|| (text.contains("No:") && text.contains("Fax"))
|| (text.contains("Contact:") && text.contains("Tel.:"))
|| text.contains("European contact:")
))
then
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "24: Redact contact information (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|| text.contains("Contact:")
|| text.contains("Alternative contact:")
|| (text.contains("No:") && text.contains("Fax"))
|| (text.contains("Contact:") && text.contains("Tel.:"))
|| text.contains("European contact:")
))
then
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Contact")
|| text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Contact")
|| text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "28: Redact AUTHOR(S) (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "30: Redact AUTHOR(S) (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:")
)
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
end
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:"))
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "33: Redact study director abbreviation"
when
section: Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
then
section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "34: Purity Hint"
when
section: Section(searchText.toLowerCase().contains("purity"))
then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
end
rule "35: Redact signatures (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "36: Redact signatures (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "43: Redact Logos (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
then
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "44: Redact Author if contained in paragraph with 'unpublished'"
when when
$section: Section() $section: Section()
$entity: Entity(type == "CBI_author") from $section.getEntities() $paragraph: Paragraph($section.sectionNumber == sectionNumber)
$publishedEntity: Entity(type == "published_information") from $section.getEntities()
$paragraph: Paragraph(sectionNumber == $section.sectionNumber)
then
if ($paragraph.containsEntity($entity) && $paragraph.containsEntity($publishedEntity) ) {
$entity.setRedaction(false); $authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber())
} $publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber())
then
$authorEntity.setRedaction(false);
end end
*/