Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-101: Detect vertebrate study row value

Browse Source
This commit is contained in:
Thierry Göckel 2020-08-10 13:30:10 +02:00
parent 06630b09d2
commit 00c96c6f57
3 changed files with 23 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -7,6 +7,7 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import lombok.Builder; import lombok.Builder;
@ -100,7 +101,7 @@ public class Section {
if (values != null) { if (values != null) {
for (String value : values) { for (String value : values) {
if (value != null && StringUtils.isNotBlank(value)) { if (StringUtils.isNotBlank(value)) {
Set<Entity> found = findEntities(value.trim(), asType); Set<Entity> found = findEntities(value.trim(), asType);
entities.addAll(found); entities.addAll(found);
} }
@ -160,14 +161,22 @@ public class Section {
} }
public void highlightCell(String reason) { public void highlightCell(String cellHeader, int ruleNumber) {
String value = tabularData.get(reason); String value = tabularData.get(cellHeader);
if (value == null) { if (value == null) {
log.warn("Could not find any data for {}.", reason); log.warn("Could not find any data for {}.", cellHeader);
} else { } else {
Entity entity = findEntities(value, "some type").iterator().next(); Set<Entity> found = findEntities(value, "must_redact");
if (CollectionUtils.isEmpty(found)) {
log.warn("Could not identify value {} in row.", value);
} else {
Entity entity = found.iterator().next();
entity.setRedaction(false); entity.setRedaction(false);
entity.setMatchedRule(ruleNumber);
entity.setRedactionReason(cellHeader);
entities.add(entity);
}
} }
} }

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java
View File

@ -117,16 +117,16 @@ public class EntityRedactionServiceTest {
"import com.iqser.red.service.redaction.v1.server.redaction.model.Section\n" + "import com.iqser.red.service.redaction.v1.server.redaction.model.Section\n" +
"\n" + "\n" +
"global Section section\n" + "global Section section\n" +
"rule \"8: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study\"\n" + "rule \"9: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study\"\n" +
" when\n" + " when\n" +
" Section(tabularData != null && tabularData.size() > 0\n" + " Section(tabularData != null && tabularData.size() > 0\n" +
" && tabularData.containsKey(\"Vertebrate\\nstudy Y/N\")\n" + " && tabularData.containsKey(\"Vertebrate\\nstudy Y/N\")\n" +
" && tabularData.get(\"Vertebrate\\nstudy Y/N\").equals(\"Y\")\n" + " && tabularData.get(\"Vertebrate\\nstudy Y/N\").equals(\"Y\")\n" +
" )\n" + " )\n" +
" then\n" + " then\n" +
" section.redact(\"name\", 8, \"Redacted because row is a vertebrate study\");\n" + " section.redact(\"name\", 9, \"Redacted because row is a vertebrate study\");\n" +
" section.redact(\"address\", 8, \"Redacted because rows is a vertebrate study\");\n" + " section.redact(\"address\", 9, \"Redacted because rows is a vertebrate study\");\n" +
" section.highlightCell(\"Vertebrate\\nstudy Y/N\");\n" + " section.highlightCell(\"Vertebrate\\nstudy Y/N\", 9);\n" +
" end"; " end";
when(rulesClient.getVersion()).thenReturn(1L); when(rulesClient.getVersion()).thenReturn(1L);
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules)); when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));
@ -149,7 +149,7 @@ public class EntityRedactionServiceTest {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc); entityRedactionService.processDocument(classifiedDoc);
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
assertThat(classifiedDoc.getEntities().get(1)).hasSize(4); // 4 out of 5 entities recognized on page 1 assertThat(classifiedDoc.getEntities().get(1)).hasSize(5); // 4 out of 5 entities recognized on page 1
} }
} }

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -106,7 +106,7 @@ rule "9: Redact Authors and Addresses in Reference Table, if it is a Vertebrate
&& tabularData.get("Vertebrate\nstudy Y/N").equals("Y") && tabularData.get("Vertebrate\nstudy Y/N").equals("Y")
) )
then then
section.redact("name", 8, "Redacted because row is a vertebrate study"); section.redact("name", 9, "Redacted because row is a vertebrate study");
section.redact("address", 8, "Redacted because rows is a vertebrate study"); section.redact("address", 9, "Redacted because rows is a vertebrate study");
section.highlightCell("Vertebrate\nstudy Y/N"); section.highlightCell("Vertebrate\nstudy Y/N", 9);
end end