diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index afe245eb..e73aec45 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -16,7 +16,6 @@ import java.util.Set; @AllArgsConstructor @EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Entity implements ReasonHolder { - private String word; private String type; private boolean redaction; @@ -26,6 +25,7 @@ public class Entity implements ReasonHolder { private List positionSequences = new ArrayList<>(); private List targetSequences; + @EqualsAndHashCode.Include private Integer start; @EqualsAndHashCode.Include @@ -66,6 +66,7 @@ public class Entity implements ReasonHolder { String headline, int matchedRule, int sectionNumber, + int paragraphNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, @@ -85,6 +86,7 @@ public class Entity implements ReasonHolder { this.headline = headline; this.matchedRule = matchedRule; this.sectionNumber = sectionNumber; + this.paragraphNumber = paragraphNumber; this.legalBasis = legalBasis; this.isDictionaryEntry = isDictionaryEntry; this.textBefore = textBefore; @@ -104,6 +106,7 @@ public class Entity implements ReasonHolder { Integer end, String headline, int sectionNumber, + int paragraphNumber, boolean isDictionaryEntry, boolean isDossierDictionaryEntry, Engine engine, @@ -115,6 +118,7 @@ public class Entity implements ReasonHolder { this.end = end; this.headline = headline; this.sectionNumber = sectionNumber; + this.paragraphNumber = paragraphNumber; this.isDictionaryEntry = isDictionaryEntry; this.isDossierDictionaryEntry = isDossierDictionaryEntry; this.engines.add(engine); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java index a3fb2d2c..8be92642 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java @@ -1,61 +1,16 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; -import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString; - -import java.util.List; - -import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; -import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; - +import lombok.AccessLevel; +import lombok.Builder; import lombok.Getter; +import lombok.experimental.FieldDefaults; @Getter +@Builder +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class Paragraph { - private final String searchText; - private final List textPositions; - private final int[] searchTextToPositions; - private final int sectionNumber; - - private final int start; - private final int end; - - - public Paragraph(TextBlock textBlock, int sectionNumber, int start, int end) { - - textPositions = textBlock.getSequences().stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList(); - this.searchTextToPositions = new int[textPositions.size()]; - this.searchText = buildString(textBlock.getSequences()); - this.start = start; - this.end = end; - this.sectionNumber = sectionNumber; - } - - public boolean containsEntity(Entity entity) { - return entity.getStart() >= start && entity.getEnd() <= end; - } - - private String buildSearchText() { - - StringBuilder searchTextBuilder = new StringBuilder(); - int numberOfNormalCharacters = 0; - - for (int i = 0; i < textPositions.size(); ++i) { - if (isNormalText(textPositions.get(i).getUnicode())) { - searchTextBuilder.append(textPositions.get(i).getUnicode()); - this.searchTextToPositions[numberOfNormalCharacters] = i; - ++numberOfNormalCharacters; - } - } - return searchTextBuilder.toString(); - } - - - private boolean isNormalText(String unicode) { - - return !unicode.equals("\n") && // - !unicode.equals(""); - - } + SearchTextWithTextPositionModel searchTextToTextPosition; + int sectionNumber; + int paragraphNumber; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java new file mode 100644 index 00000000..c849da04 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java @@ -0,0 +1,22 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import java.util.List; +import java.util.Map; + +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.experimental.FieldDefaults; + +@Builder +@Getter +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class SearchTextWithTextPositionModel { + + String searchText; + Map stringCoordsToTextPositionCoords; + List lineBreaksStringCoords; + List textPositionSequences; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java index 7cd78e6d..1de00703 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java @@ -1,17 +1,18 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils; -import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; - import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; +import com.dslplatform.json.JsonAttribute; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; +import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils; +import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; + import lombok.Getter; public class SearchableText { @@ -184,30 +185,13 @@ public class SearchableText { return stringRepresentation; } - -// public List getParagraphStrings(){ -// -// } - - public static String buildString(List sequences) { StringBuilder sb = new StringBuilder(); - - TextPositionSequence previous = null; for (TextPositionSequence word : sequences) { - - if (previous != null) { - if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) { - sb.append('\n'); - } else { - sb.append(' '); - } - } sb.append(word.toString()); - previous = word; + sb.append(' '); } - return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " "); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index f89345c8..7e68a938 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1363,6 +1363,7 @@ public class Section { value.getRowSpanStart() + word.length(), headline, sectionNumber, + -1, false, false, Engine.RULE, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 10f02ff5..32e073f0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -4,7 +4,7 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -22,6 +22,8 @@ import org.kie.api.builder.KieFileSystem; import org.kie.api.builder.KieModule; import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieSession; +import org.kie.api.runtime.rule.QueryResults; +import org.kie.api.runtime.rule.QueryResultsRow; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.server.client.RulesClient; @@ -88,19 +90,29 @@ public class DroolsExecutionService { @Timed("redactmanager_executeRules") - public List
executeRules(KieContainer kieContainer, List
sections, List paragraphs, Dictionary dictionary) { + public List executeRules(KieContainer kieContainer, List
sections, List paragraphs, Dictionary dictionary) { - Set entities = new HashSet<>(); KieSession kieSession = kieContainer.newKieSession(); + kieSession.setGlobal("dictionary", dictionary); sections.forEach(kieSession::insert); paragraphs.forEach(kieSession::insert); kieSession.fireAllRules(); + List entities = getEntities(kieSession); kieSession.dispose(); - return sections; + return entities; } + public List getEntities(KieSession ks) { + List entities = new LinkedList<>(); + QueryResults entitiesResult = ks.getQueryResults("getEntities"); + for (QueryResultsRow resultsRow : entitiesResult) { + entities.add((Entity) resultsRow.get("$result")); + } + return entities; + } + public KieContainer updateRules(String dossierTemplateId) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java new file mode 100644 index 00000000..a059a63a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java @@ -0,0 +1,81 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel; + +@Service +public class SearchTextWithTextPositionFactory { + + public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List sequences) { + + Map stringIdxToPositionIdx = new HashMap<>(); + List lineBreaksStringIdx = new LinkedList<>(); + StringBuilder sb = new StringBuilder(); + + int stringIdx = 0; + int positionIdx = 0; + + String currentUnicode; + String previousUnicode = " "; + + for (TextPositionSequence word : sequences) { + for (int i = 0; i < word.getTextPositions().size(); ++i) { + + currentUnicode = word.getTextPositions().get(i).getUnicode(); + + if (isLineBreak(currentUnicode)) { + lineBreaksStringIdx.add(stringIdx); + } else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && // + !isHyphenLinebreak(currentUnicode)) { + + sb.append(currentUnicode); + stringIdxToPositionIdx.put(stringIdx, positionIdx); + ++stringIdx; + } + + previousUnicode = currentUnicode; + ++positionIdx; + } + + previousUnicode = " "; + sb.append(previousUnicode); + stringIdxToPositionIdx.put(stringIdx, positionIdx); + ++stringIdx; + } + + return SearchTextWithTextPositionModel.builder() + .searchText(sb.toString().stripTrailing()) + .lineBreaksStringCoords(lineBreaksStringIdx) + .stringCoordsToTextPositionCoords(stringIdxToPositionIdx) + .textPositionSequences(sequences) + .build(); + } + + + private boolean isLineBreak(String currentUnicode) { + + return Objects.equals(currentUnicode, "\n"); + } + + + private boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) { + + return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " "); + } + + + private boolean isHyphenLinebreak(String unicodeCharacter) { + + return unicodeCharacter.matches("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R"); + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java index 4cfb5e6a..c1270672 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; -import lombok.AllArgsConstructor; -import lombok.Data; - -import org.ahocorasick.trie.Trie; - import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -12,6 +7,10 @@ import java.util.Locale; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.ahocorasick.trie.Trie; + +import lombok.Data; + @Data public class SearchImplementation { @@ -94,13 +93,7 @@ public class SearchImplementation { } } - - @Data - @AllArgsConstructor - public static class MatchPosition { - - private int startIndex; - private int endIndex; + public record MatchPosition(int startIndex, int endIndex) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java index 6752c4bb..e01636ec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java @@ -17,8 +17,8 @@ public class EntitySearchUtilsTest { public void testNestedEntitiesRemoval() { Set entities = new HashSet<>(); - Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); entities.add(nested); entities.add(nesting); EntitySearchUtils.removeEntitiesContainedInLarger(entities); @@ -40,14 +40,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -73,14 +73,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -105,14 +105,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -137,15 +137,15 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); foundEntities.add(foundEntities3); @@ -171,16 +171,16 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntitiesOverlap1); foundEntities.add(foundEntitiesOverlap2); foundEntities.add(foundEntitiesSubset1); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 42b44ab3..8695d99f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1,390 +1,67 @@ package drools -import com.iqser.red.service.redaction.v1.server.redaction.model.* +import static java.lang.String.format; +import java.util.List; +import java.util.LinkedList; +import java.util.HashSet; +import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.model.Engine; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import java.util.Set; + +global Dictionary dictionary + +query "getEntities" + $result: Entity() +end // --------------------------------------- AI rules ------------------------------------------------------------------- -rule "-1: find entities from dictionary" +rule "0: find entities from entry dictionary" salience 100 - no-loop true when - section: Section(!text.isEmpty()) + $section: Section() + $paragraph: Paragraph(sectionNumber == $section.sectionNumber) + $entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition()) then - section.findDictionaryEntities(); - update(section); - end - -rule "0: Add CBI_author from ai" - when - section: Section(aiMatchesType("CBI_author")) - then - section.addAiEntities("CBI_author", "CBI_author"); - end - -rule "0: Combine address parts from ai to CBI_address (org is mandatory)" - when - section: Section(aiMatchesType("ORG")) - then - section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); - end - -rule "0: Combine address parts from ai to CBI_address (street is mandatory)" - when - section: Section(aiMatchesType("STREET")) - then - section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); - end - -rule "0: Combine address parts from ai to CBI_address (city is mandatory)" - when - section: Section(aiMatchesType("CITY")) - then - section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false); - end - - -// --------------------------------------- CBI rules ------------------------------------------------------------------- - -rule "1: Redact CBI Authors (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) - then - section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "2: Redact CBI Authors (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) - then - section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "3: Redact not CBI Address (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) - then - section.redactNot("CBI_address", 3, "Address found for non vertebrate study"); - section.ignoreRecommendations("CBI_address"); - end - -rule "4: Redact CBI Address (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) - then - section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "5: Do not redact genitive CBI_author" - when - section: Section(matchesType("CBI_author")) - then - section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); - end - - -rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) - then - section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) - then - section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "14: Redact and add recommendation for et al. author (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "15: Redact and add recommendation for et al. author (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "16: Add recommendation for Addresses in Test Organism sections" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:")) - then - section.recommendLineAfter("Source:", "CBI_address"); - end - -rule "17: Add recommendation for Addresses in Test Animals sections" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source")) - then - section.recommendLineAfter("Source", "CBI_address"); + List eps = new LinkedList<>(); + eps.add($entityPositionSequence); + Set engineSet = new HashSet<>(); + engineSet.add(Engine.DICTIONARY); + Set empty = new HashSet<>(); + insert(new Entity("CBI_author", + "custom", + true, + format("Found in Dictionary: %s", "custom"), + eps, + $section.getHeadline(), + 0, + $section.getSectionNumber(), + $paragraph.getParagraphNumber(), + "", + true, + "", + "", + -1, + -1, + false, + engineSet, + empty, + EntityType.ENTITY + )); end /* - -rule "18: Do not redact Names and Addresses if Published Information found" - when - section: Section(matchesType("published_information")) - then - section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found"); - section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found"); - end - -*/ - -// --------------------------------------- PII rules ------------------------------------------------------------------- - - -rule "19: Redacted PII Personal Identification Information (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) - then - section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "20: Redacted PII Personal Identification Information (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) - then - section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "21: Redact Emails by RegEx (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) - then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "22: Redact Emails by RegEx (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) - then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "23: Redact contact information (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") - || text.contains("Contact:") - || text.contains("Alternative contact:") - || (text.contains("No:") && text.contains("Fax")) - || (text.contains("Contact:") && text.contains("Tel.:")) - || text.contains("European contact:") - )) - then - section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "24: Redact contact information (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") - || text.contains("Contact:") - || text.contains("Alternative contact:") - || (text.contains("No:") && text.contains("Fax")) - || (text.contains("Contact:") && text.contains("Tel.:")) - || text.contains("European contact:") - )) - then - section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( - text.contains("Contact") - || text.contains("Telephone") - || text.contains("Phone") - || text.contains("Fax") - || text.contains("Tel") - || text.contains("Ter") - || text.contains("Mobile") - || text.contains("Fel") - || text.contains("Fer") - )) - then - section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( - text.contains("Contact") - || text.contains("Telephone") - || text.contains("Phone") - || text.contains("Fax") - || text.contains("Tel") - || text.contains("Ter") - || text.contains("Mobile") - || text.contains("Fel") - || text.contains("Fer") - )) - then - section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "27: Redact AUTHOR(S) (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("COMPLETION DATE:") - && !searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "28: Redact AUTHOR(S) (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("COMPLETION DATE:") - && !searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "29: Redact AUTHOR(S) (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "30: Redact AUTHOR(S) (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("PERFORMING LABORATORY:") - ) - then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); - end - -rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("PERFORMING LABORATORY:")) - then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "33: Redact study director abbreviation" - when - section: Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML"))) - then - section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - - -// --------------------------------------- other rules ------------------------------------------------------------------- - -rule "34: Purity Hint" - when - section: Section(searchText.toLowerCase().contains("purity")) - then - section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); - end - - - -rule "35: Redact signatures (Non vertebrate study)" - when - section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) - then - section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "36: Redact signatures (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) - then - section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "43: Redact Logos (Vertebrate study)" - when - section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo")) - then - section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "44: Redact Author if contained in paragraph with 'unpublished'" +rule "44: Don't redact Author if contained in paragraph with published information" when $section: Section() - $entity: Entity(type == "CBI_author") from $section.getEntities() - $publishedEntity: Entity(type == "published_information") from $section.getEntities() - $paragraph: Paragraph(sectionNumber == $section.sectionNumber) - then - if ($paragraph.containsEntity($entity) && $paragraph.containsEntity($publishedEntity) ) { + $paragraph: Paragraph($section.sectionNumber == sectionNumber) - $entity.setRedaction(false); - } + $authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber()) + $publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber()) + then + $authorEntity.setRedaction(false); end +*/ \ No newline at end of file