diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index e83694a2..8931f6ed 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -1,15 +1,19 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; -import com.iqser.red.service.redaction.v1.model.Engine; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; - -import lombok.*; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import com.iqser.red.service.redaction.v1.model.Engine; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + @Data @Builder @NoArgsConstructor @@ -38,6 +42,9 @@ public class Entity implements ReasonHolder { @EqualsAndHashCode.Include private int sectionNumber; + @EqualsAndHashCode.Include + private int paragraphNumber; + private boolean isDictionaryEntry; private String textBefore; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java new file mode 100644 index 00000000..a3fb2d2c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java @@ -0,0 +1,61 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString; + +import java.util.List; + +import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; +import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; + +import lombok.Getter; + +@Getter +public class Paragraph { + + private final String searchText; + private final List textPositions; + private final int[] searchTextToPositions; + private final int sectionNumber; + + private final int start; + private final int end; + + + public Paragraph(TextBlock textBlock, int sectionNumber, int start, int end) { + + textPositions = textBlock.getSequences().stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList(); + this.searchTextToPositions = new int[textPositions.size()]; + this.searchText = buildString(textBlock.getSequences()); + this.start = start; + this.end = end; + this.sectionNumber = sectionNumber; + } + + public boolean containsEntity(Entity entity) { + return entity.getStart() >= start && entity.getEnd() <= end; + } + + private String buildSearchText() { + + StringBuilder searchTextBuilder = new StringBuilder(); + int numberOfNormalCharacters = 0; + + for (int i = 0; i < textPositions.size(); ++i) { + if (isNormalText(textPositions.get(i).getUnicode())) { + searchTextBuilder.append(textPositions.get(i).getUnicode()); + this.searchTextToPositions[numberOfNormalCharacters] = i; + ++numberOfNormalCharacters; + } + } + return searchTextBuilder.toString(); + } + + + private boolean isNormalText(String unicode) { + + return !unicode.equals("\n") && // + !unicode.equals(""); + + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index da19af7e..c0e92faa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -27,7 +27,6 @@ import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.model.SectionArea; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; -import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService; @@ -39,7 +38,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; -import io.micrometer.core.annotation.Timed; import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -56,7 +54,8 @@ public class Section { @Builder.Default private Map> localDictionaryAdds = new HashMap<>(); - private Set entities; + @Builder.Default + private Set entities = new HashSet<>(); private Set nerEntities; @@ -97,12 +96,10 @@ public class Section { private RedactionServiceSettings redactionServiceSettings; - @Deprecated @SuppressWarnings("unused") @ThenAction public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) { - redactOrRecommendAiEntities(type, asType, false, 0, null, null); } @@ -1775,14 +1772,4 @@ public class Section { entities = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions); nerEntities = EntitySearchUtils.clearAndFindPositions(nerEntities, searchableText, dictionary, manualRedactions); } - - - - } - - - - - - diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java deleted file mode 100644 index d164e7b9..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; - -import java.util.List; - -import lombok.AllArgsConstructor; -import lombok.Data; - -@Data -@AllArgsConstructor -public class SectionSearchableTextPair { - - private Section section; - private SearchableText searchableText; - private List cellStarts; - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 6905fb63..3a21f8ac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -1,11 +1,13 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; -import com.iqser.red.service.redaction.v1.server.redaction.model.Section; - -import io.micrometer.core.annotation.Timed; -import lombok.RequiredArgsConstructor; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.kie.api.KieServices; @@ -16,11 +18,15 @@ import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieSession; import org.springframework.stereotype.Service; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; +import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; +import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph; +import com.iqser.red.service.redaction.v1.server.redaction.model.Section; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor @@ -45,15 +51,16 @@ public class DroolsExecutionService { @Timed("redactmanager_executeRules") - public Section executeRules(KieContainer kieContainer, Section section) { + public List
executeRules(KieContainer kieContainer, List
sections, List paragraphs, Dictionary dictionary) { + Set entities = new HashSet<>(); KieSession kieSession = kieContainer.newKieSession(); - kieSession.setGlobal("section", section); - kieSession.insert(section); + sections.forEach(kieSession::insert); + paragraphs.forEach(kieSession::insert); kieSession.fireAllRules(); kieSession.dispose(); - return section; + return sections; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index df9a4c66..cc0d1461 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,33 +1,45 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import static com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText.buildString; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.kie.api.runtime.KieContainer; +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entities; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult; +import com.iqser.red.service.redaction.v1.server.redaction.model.Image; +import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; +import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; +import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; -import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; - -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @Slf4j @Service @RequiredArgsConstructor @@ -44,7 +56,7 @@ public class EntityRedactionService { if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) { - if(!findEntitiesResult.getAddedFileAttributes().isEmpty()) { + if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) { //AnalyzeRequest provides immutable list. List mergedFileAttributes = new ArrayList<>(); mergedFileAttributes.addAll(analyzeRequest.getFileAttributes()); @@ -53,7 +65,14 @@ public class EntityRedactionService { } Map> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary); - FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage, nerEntities); + FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, + dictionary, + kieContainer, + analyzeRequest, + true, + hintsPerSectionNumber, + imagesPerPage, + nerEntities); EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary); EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities()); } @@ -66,15 +85,16 @@ public class EntityRedactionService { public FindEntitiesResult findEntities(List reanalysisSections, - Dictionary dictionary, - KieContainer kieContainer, - AnalyzeRequest analyzeRequest, - boolean local, - Map> hintsPerSectionNumber, - Map> imagesPerPage, - NerEntities nerEntities) { + Dictionary dictionary, + KieContainer kieContainer, + AnalyzeRequest analyzeRequest, + boolean local, + Map> hintsPerSectionNumber, + Map> imagesPerPage, + NerEntities nerEntities) { + + List
sections = new ArrayList<>(reanalysisSections.size()); - List sectionSearchableTextPairs = new ArrayList<>(); for (SectionText reanalysisSection : reanalysisSections) { Set nerFound = new HashSet<>(); @@ -83,8 +103,7 @@ public class EntityRedactionService { } log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages()); - - sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() + sections.add(Section.builder() .isLocal(false) .dictionaryTypes(dictionary.getTypes()) .nerEntities(nerFound) @@ -102,25 +121,39 @@ public class EntityRedactionService { .isInTable(reanalysisSection.isTable()) .redactionServiceSettings(redactionServiceSettings) .cellStarts(reanalysisSection.getCellStarts()) - .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts())); + .build()); } - Set addedFileAttributes = new HashSet<>(); Set entities = new HashSet<>(); - sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { + sections.forEach(section -> { - if(!addedFileAttributes.isEmpty()) { + if (!addedFileAttributes.isEmpty()) { //Section.Builder provides immutable list. List mergedFileAttributes = new ArrayList<>(); - mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes()); + mergedFileAttributes.addAll(section.getAddedFileAttributes()); mergedFileAttributes.addAll(addedFileAttributes); - sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes); + section.setFileAttributes(mergedFileAttributes); } + }); - Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); + List paragraphs = new ArrayList<>(); + for (var reanalysisSection : reanalysisSections) { + int start = 0; + int end; + for (var textBlock : reanalysisSection.getTextBlocks()) { + int length = buildString(textBlock.getSequences()).length(); + end = start + length; + paragraphs.add(new Paragraph(textBlock, reanalysisSection.getSectionNumber(), start, end)); + start = end + 1; + } + } + + List
analysedSections = droolsExecutionService.executeRules(kieContainer, sections, paragraphs, dictionary); + + analysedSections.forEach(analysedSection -> { addedFileAttributes.addAll(analysedSection.getAddedFileAttributes()); EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities()); @@ -130,13 +163,19 @@ public class EntityRedactionService { .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null) .collect(Collectors.toSet()); - if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) { + if (analysedSection.getCellStarts() != null && !analysedSection.getCellStarts().isEmpty()) { SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, - sectionSearchableTextPair.getSearchableText(), + analysedSection.getSearchableText(), dictionary, - sectionSearchableTextPair.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords()); + analysedSection.getCellStarts(), + redactionServiceSettings.getSurroundingWordsOffsetWindow(), + redactionServiceSettings.getNumberOfSurroundingWords()); } else { - SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords()); + SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, + analysedSection.getSearchableText(), + dictionary, + redactionServiceSettings.getSurroundingWordsOffsetWindow(), + redactionServiceSettings.getNumberOfSurroundingWords()); } entities.addAll(analysedSection.getEntities()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 510e2276..8203983e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -99,7 +99,7 @@ public final class EntitySearchUtils { List orderedEntities = entitiesByWord.get(word).stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList()); Entity firstEntity = orderedEntities.get(0); - List positionSequences = text.getSequences(firstEntity.getWord().trim(), + List positionSequences = text.getSequences(word.trim(), dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 58bb3cb6..df01eddc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -364,7 +364,7 @@ public class RedactionIntegrationTest { @Test public void titleExtraction() throws IOException { - AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); + AnalyzeRequest request = prepareStorage("files/Trinexapac/93 Trinexapac-ethyl_RAR_03_Volume_3CA_B-1_2017-03-31.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -1111,7 +1111,7 @@ public class RedactionIntegrationTest { System.out.println("classificationTest"); - AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); + AnalyzeRequest request = prepareStorage("files/Trinexapac/93 Trinexapac-ethyl_RAR_03_Volume_3CA_B-1_2017-03-31.pdf"); RedactionRequest redactionRequest = RedactionRequest.builder() .dossierId(request.getDossierId()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 5d6c0c6a..42b44ab3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1,413 +1,390 @@ package drools -import com.iqser.red.service.redaction.v1.server.redaction.model.Section - -global Section section +import com.iqser.red.service.redaction.v1.server.redaction.model.* -// --------------------------------------- CBI rules ------------------------------------------------------------------- -//rule "0: Expand CBI Authors with firstname initials" -// when -// Section(matchesType("CBI_author")) -// then -// section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1); -// end - -rule "-1: Find dictionary entries" +// --------------------------------------- AI rules ------------------------------------------------------------------- +rule "-1: find entities from dictionary" + salience 100 + no-loop true when - Section(findDictionaryEntities()); + section: Section(!text.isEmpty()) then - end - + section.findDictionaryEntities(); + update(section); + end rule "0: Add CBI_author from ai" when - Section(aiMatchesType("CBI_author")) + section: Section(aiMatchesType("CBI_author")) then section.addAiEntities("CBI_author", "CBI_author"); end -rule "0: Combine ai types CBI_author from ai" +rule "0: Combine address parts from ai to CBI_address (org is mandatory)" when - Section(aiMatchesType("ORG")) + section: Section(aiMatchesType("ORG")) then section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); end -rule "0: Expand CBI Authors with firstname initials" +rule "0: Combine address parts from ai to CBI_address (street is mandatory)" when - Section(matchesType("CBI_author")) + section: Section(aiMatchesType("STREET")) then - section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); + section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); end -rule "0: Expand CBI_author and PII matches with salutation prefix" +rule "0: Combine address parts from ai to CBI_address (city is mandatory)" when - Section((matchesType("CBI_author") || matchesType("PII")) && ( - searchText.contains("Mr") - || searchText.contains("Mrs") - || searchText.contains("Ms") - || searchText.contains("Miss") - || searchText.contains("Sir") - || searchText.contains("Madam") - || searchText.contains("Madame") - || searchText.contains("Mme") - )) + section: Section(aiMatchesType("CITY")) then - section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0); - section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0); + section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false); end -rule "1: Redacted because Section contains Vertebrate" +// --------------------------------------- CBI rules ------------------------------------------------------------------- + +rule "1: Redact CBI Authors (Non vertebrate study)" when - Section(matchesType("vertebrate")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) then - section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "2: Redact CBI Authors (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + then + section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "2: Not Redacted because Section contains no Vertebrate" +rule "3: Redact not CBI Address (Non vertebrate study)" when - Section(!matchesType("vertebrate")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) then - section.redactNot("CBI_author", 2, "No Vertebrate found"); - section.redactNot("CBI_address", 2, "No Vertebrate found"); + section.redactNot("CBI_address", 3, "Address found for non vertebrate study"); + section.ignoreRecommendations("CBI_address"); + end + +rule "4: Redact CBI Address (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + then + section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "3: Do not redact Names and Addresses if no redaction Indicator is contained" +rule "5: Do not redact genitive CBI_author" when - Section(matchesType("vertebrate"), matchesType("no_redaction_indicator")) + section: Section(matchesType("CBI_author")) then - section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found"); - section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found"); + section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); end -rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained" +rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)" when - Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) then - section.redact("CBI_author", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.redact("CBI_address", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "5: Do not redact Names and Addresses if no redaction Indicator is contained" +rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)" when - Section(matchesType("vertebrate"), matchesType("published_information")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactNotAndReference("CBI_author","published_information", 5, "Vertebrate and Published Information found"); - section.redactNotAndReference("CBI_address","published_information", 5, "Vertebrate and Published Information found"); + section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "6: Not redacted because Vertebrate Study = N" +rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)" when - Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) then - section.redactNotCell("Author(s)", 6, "CBI_author", true, "Not redacted because row is not a vertebrate study"); - section.redactNot("CBI_author", 6, "Not redacted because row is not a vertebrate study"); - section.redactNot("CBI_address", 6, "Not redacted because row is not a vertebrate study"); - section.highlightCell("Vertebrate study Y/N", 6, "hint_only"); + section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) + then + section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "14: Redact and add recommendation for et al. author (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) + then + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "15: Redact and add recommendation for et al. author (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) + then + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "7: Redact if must redact entry is found" +rule "16: Add recommendation for Addresses in Test Organism sections" when - Section(matchesType("must_redact")) + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:")) then - section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + section.recommendLineAfter("Source:", "CBI_address"); end - -rule "8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study" +rule "17: Add recommendation for Addresses in Test Animals sections" when - Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes")) + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source")) then - section.redactCell("Author(s)", 8, "CBI_author", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.redact("CBI_address", 8, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.highlightCell("Vertebrate study Y/N", 8, "must_redact"); + section.recommendLineAfter("Source", "CBI_address"); end +/* -rule "9: Redact sponsor company" +rule "18: Do not redact Names and Addresses if Published Information found" when - Section(searchText.toLowerCase().contains("batches produced at")) + section: Section(matchesType("published_information")) then - section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 9, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.addHintAnnotation("batches produced at", "must_redact"); + section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found"); + section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found"); end - -rule "10: Redact determination of residues" - when - Section(( - searchText.toLowerCase.contains("determination of residues") || - searchText.toLowerCase.contains("determination of total residues") - ) && ( - searchText.toLowerCase.contains("livestock") || - searchText.toLowerCase.contains("live stock") || - searchText.toLowerCase.contains("tissue") || - searchText.toLowerCase.contains("tissues") || - searchText.toLowerCase.contains("liver") || - searchText.toLowerCase.contains("muscle") || - searchText.toLowerCase.contains("bovine") || - searchText.toLowerCase.contains("ruminant") || - searchText.toLowerCase.contains("ruminants") - )) - then - section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.addHintAnnotation("determination of residues", "must_redact"); - section.addHintAnnotation("livestock", "must_redact"); - section.addHintAnnotation("live stock", "must_redact"); - section.addHintAnnotation("tissue", "must_redact"); - section.addHintAnnotation("tissues", "must_redact"); - section.addHintAnnotation("liver", "must_redact"); - section.addHintAnnotation("muscle", "must_redact"); - section.addHintAnnotation("bovine", "must_redact"); - section.addHintAnnotation("ruminant", "must_redact"); - section.addHintAnnotation("ruminants", "must_redact"); - end - - -rule "11: Redact if CTL/* or BL/* was found" - when - Section(searchText.contains("CTL/") || searchText.contains("BL/")) - then - section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - section.addHintAnnotation("CTL", "must_redact"); - section.addHintAnnotation("BL", "must_redact"); - end - - -rule "12: Redact and add recommendation for et al. author" - when - Section(searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - end - - -rule "13: Add recommendation for Addresses in Test Organism sections" - when - Section(searchText.contains("Species:") && searchText.contains("Source:")) - then - section.recommendLineAfter("Source:", "CBI_address"); - end - - -rule "14: Add recommendation for Addresses in Test Animals sections" - when - Section(searchText.contains("Species") && searchText.contains("Source")) - then - section.recommendLineAfter("Source", "CBI_address"); - end +*/ // --------------------------------------- PII rules ------------------------------------------------------------------- -rule "14: Redacted PII Personal Identification Information" +rule "19: Redacted PII Personal Identification Information (Non vertebrate study)" when - Section(matchesType("PII")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) then - section.redact("PII", 14, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "20: Redacted PII Personal Identification Information (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) + then + section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "15: Redact Emails by RegEx" +rule "21: Redact Emails by RegEx (Non vertebrate study)" when - Section(searchText.contains("@")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) then - section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 15, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "22: Redact Emails by RegEx (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) + then + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "16: Redact contact information" +rule "23: Redact contact information (Non vertebrate study)" when - Section(text.contains("Contact point:") - || text.contains("Phone:") - || text.contains("Fax:") - || text.contains("Tel.:") - || text.contains("Tel:") - || text.contains("E-mail:") - || text.contains("Email:") - || text.contains("e-mail:") - || text.contains("E-mail address:") - || text.contains("Alternative contact:") - || text.contains("Telephone number:") - || text.contains("Telephone No:") - || text.contains("Fax number:") - || text.contains("Telephone:") - || text.contains("European contact:")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") + || text.contains("Contact:") + || text.contains("Alternative contact:") + || (text.contains("No:") && text.contains("Fax")) + || (text.contains("Contact:") && text.contains("Tel.:")) + || text.contains("European contact:") + )) then - section.redactLineAfter("Contact point:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Phone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Fax:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Tel:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("E-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Email:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("e-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("E-mail address:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Alternative contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Telephone number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Telephone No:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Fax number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Telephone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactBetween("No:", "Fax", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactBetween("Contact:", "Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("European contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end - -rule "17: Redact contact information if applicant is found" +rule "24: Redact contact information (Vertebrate study)" when - Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")) + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") + || text.contains("Contact:") + || text.contains("Alternative contact:") + || (text.contains("No:") && text.contains("Fax")) + || (text.contains("Contact:") && text.contains("Tel.:")) + || text.contains("European contact:") + )) then - section.redactLineAfter("Contact point:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Phone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Fax:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Tel:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("E-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Email:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("e-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("E-mail address:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Alternative contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Telephone number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Telephone No:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Fax number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Telephone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactBetween("No:", "Fax", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactBetween("Contact:", "Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("European contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( + text.contains("Contact") + || text.contains("Telephone") + || text.contains("Phone") + || text.contains("Fax") + || text.contains("Tel") + || text.contains("Ter") + || text.contains("Mobile") + || text.contains("Fel") + || text.contains("Fer") + )) + then + section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( + text.contains("Contact") + || text.contains("Telephone") + || text.contains("Phone") + || text.contains("Fax") + || text.contains("Tel") + || text.contains("Ter") + || text.contains("Mobile") + || text.contains("Fel") + || text.contains("Fer") + )) + then + section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "27: Redact AUTHOR(S) (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("AUTHOR(S):") + && searchText.contains("COMPLETION DATE:") + && !searchText.contains("STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "28: Redact AUTHOR(S) (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("AUTHOR(S):") + && searchText.contains("COMPLETION DATE:") + && !searchText.contains("STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "18: Redact contact information if Producer is found" +rule "29: Redact AUTHOR(S) (Non vertebrate study)" when - Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("AUTHOR(S):") + && searchText.contains("STUDY COMPLETION DATE:") + ) then - section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Telephone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Phone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Fax:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("E-mail:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Fax number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Telephone number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactLineAfter("Tel:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - section.redactBetween("No:", "Fax", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "30: Redact AUTHOR(S) (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("AUTHOR(S):") + && searchText.contains("STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "19: Redact AUTHOR(S)" +rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" when - Section(searchText.contains("AUTHOR(S):") && fileAttributeByPlaceholderEquals("{fileattributes.vertebrateStudy}", "true")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("PERFORMING LABORATORY:") + ) then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); end - -rule "20: Redact PERFORMING LABORATORY" +rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" when - Section(searchText.contains("PERFORMING LABORATORY:")) + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("PERFORMING LABORATORY:")) then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 20, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end - -rule "21: Redact On behalf of Sequani Ltd.:" +rule "33: Redact study director abbreviation" when - Section(searchText.contains("On behalf of Sequani Ltd.: Name Title")) + section: Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML"))) then - section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 21, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - end + section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end -rule "22: Redact On behalf of Syngenta Ltd.:" - when - Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title")) - then - section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - end - // --------------------------------------- other rules ------------------------------------------------------------------- -rule "25: Redact Purity" +rule "34: Purity Hint" when - Section(searchText.contains("purity")) + section: Section(searchText.toLowerCase().contains("purity")) then - section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"); + section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); end -rule "26: Redact signatures" + +rule "35: Redact signatures (Non vertebrate study)" when - Section(matchesImageType("signature")) + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) then - section.redactImage("signature", 26, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "36: Redact signatures (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) + then + section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "27: Redact formula" +rule "43: Redact Logos (Vertebrate study)" when - Section(matchesImageType("formula")) + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo")) then - section.redactImage("formula", 27, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end - -rule "28: Redact Logos" +rule "44: Redact Author if contained in paragraph with 'unpublished'" when - Section(matchesImageType("logo")) + $section: Section() + $entity: Entity(type == "CBI_author") from $section.getEntities() + $publishedEntity: Entity(type == "published_information") from $section.getEntities() + $paragraph: Paragraph(sectionNumber == $section.sectionNumber) then - section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + if ($paragraph.containsEntity($entity) && $paragraph.containsEntity($publishedEntity) ) { + + $entity.setRedaction(false); + } end - - -rule "29: Redact Dossier Redactions" - when - Section(matchesType("dossier_redactions")) - then - section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - end - -rule "30: Ignore dossier_redactions if confidential" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions")); - then - section.ignore("dossier_redactions"); - end - -// ex. "New Rules for PAD" - "Annex A" - page 21, page 35 (table without header), page 38 (in-text) -// https://www.regexplanet.com/share/index.html?share=yyyypb71xkr -rule "101: Redact CAS numbers" - when - Section(hasTableHeader("Sample #")) - then - section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - end - -rule "102: Guidelines FileAttributes" - when - Section((text.contains("DATA REQUIREMENT(S):") || text.contains("TEST GUIDELINE(S):")) && (text.contains("OECD") || text.contains("EPA") || text.contains("OPPTS"))) - then - section.addFileAttribute("OECD Number", "OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", false, 0); - end - -rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)" - when - Section(hasTableHeader("h5.1")) - then - section.redactCell("h5.1", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end \ No newline at end of file